` after being safely escaped.\n\n Usage:\n Load the tag library, then invoke the tag with a name and optional\n HTML attributes.\n\n Template:\n\n {% load visualization %}\n {% concordia_visualization \"daily-activity\"\n style=\"float:left;\" class=\"chart\" data-role=\"viz\" %}\n\n Output:\n\n
\n \n
\n\n Notes:\n - The `class` attribute you pass is appended to\n `visualization-container`.\n - All attribute names and values are escaped.\n - This tag does not include any `\",\n }\n data[\"description\"] += \"
Arm \"\n form = SanitizedDescriptionAdminForm(data)\n self.assertTrue(form.is_valid())\n self.assertEqual(form.clean_short_description(), short_description)\n self.assertEqual(form.clean_description(), \"
Arm \")\n\n\nclass ClearCacheFormTests(TestCase):\n @override_settings(\n CACHES={\n \"default\": {\n \"BACKEND\": \"django.core.cache.backends.dummy.DummyCache\",\n },\n \"view_cache\": {\n \"BACKEND\": \"django.core.cache.backends.dummy.DummyCache\",\n },\n }\n )\n def test_cache_name_choices(self):\n choices = get_cache_name_choices()\n choice_names = [name for name, description in choices]\n self.assertNotIn(\"default\", choice_names)\n self.assertIn(\"view_cache\", choice_names)\n"
},
{
"path": "concordia/tests/test_admin_views.py",
"content": "import copy\nimport json\nfrom functools import wraps\nfrom http import HTTPStatus\nfrom io import BytesIO\nfrom unittest import mock\n\nfrom django.contrib.messages import get_messages\nfrom django.core.files.uploadedfile import SimpleUploadedFile\nfrom django.test import RequestFactory, TestCase\nfrom django.urls import reverse\nfrom django.utils import timezone\nfrom django.utils.text import slugify\nfrom django.utils.timezone import now\n\nfrom concordia.admin.views import SerializedObjectView\nfrom concordia.models import Campaign, Project, TranscriptionStatus\nfrom concordia.tests.utils import (\n CreateTestUsers,\n StreamingTestMixin,\n create_asset,\n create_campaign,\n create_card,\n create_item,\n create_project,\n create_site_report,\n create_transcription,\n)\nfrom concordia.utils import get_anonymous_user\nfrom importer.tests.utils import create_import_asset\n\n\nclass TestProjectLevelExportView(CreateTestUsers, TestCase):\n def setUp(self):\n self.login_user(is_staff=True, is_superuser=True)\n self.url = reverse(\"admin:project-level-export\")\n self.asset = create_asset(download_url=\"http://example.com/1234.jpg\")\n self.asset2 = create_asset(\n slug=\"asset-2\",\n item=self.asset.item,\n download_url=\"http://example.com/5678.jpg\",\n )\n self.asset3 = create_asset(\n slug=\"asset-3\",\n item=self.asset.item,\n download_url=\"http://example.com/9012.jpg\",\n )\n\n def test_get(self):\n response = self.client.get(self.url)\n self.assertEqual(response.status_code, 200)\n self.assertContains(\n response, f\"
{self.asset.item.project.campaign.title} \", html=True\n )\n\n def test_get_campaign(self):\n response = self.client.get(\n self.url, {\"id\": self.asset.item.project.campaign.id}\n )\n self.assertContains(\n response, f\"
{self.asset.item.project.title} \", html=True\n )\n\n def test_post(self):\n with mock.patch(\"exporter.views.boto3.resource\", autospec=True) as bucket_mock:\n # The parameter is 'project_name', but it actually expects the project id.\n response = self.client.post(\n f\"{self.url}?slug={self.asset.item.project.campaign.slug}\",\n {\"project_name\": f\"{self.asset.item.project.id}\"},\n )\n self.assertEqual(response.status_code, 200)\n self.assertEqual(response[\"Content-Type\"], \"application/zip\")\n self.assertFalse(bucket_mock.called)\n\n\nclass TestFunctionBasedViews(CreateTestUsers, TestCase, StreamingTestMixin):\n def test_admin_bulk_import_review(self):\n self.login_user(is_staff=True, is_superuser=True)\n self.assertTrue(self.user.is_active)\n self.assertTrue(self.user.is_staff)\n self.assertTrue(self.user.is_superuser)\n path = reverse(\"admin:bulk-review\")\n response = self.client.get(path)\n self.assertEqual(response.status_code, 200)\n\n data = {}\n response = self.client.post(path, data=data)\n self.assertEqual(response.status_code, 200)\n\n def test_admin_site_report_view(self):\n self.login_user(is_staff=True, is_superuser=True)\n mocked_datetime = timezone.now()\n mocked_datetime_formatted = mocked_datetime.isoformat()\n with mock.patch(\"django.utils.timezone.now\") as now_mocked:\n now_mocked.return_value = mocked_datetime\n create_site_report()\n\n response = self.client.get(reverse(\"admin:site-report\"))\n self.assertEqual(response.status_code, 200)\n content = self.get_streaming_content(response).split(b\"\\r\\n\")\n self.assertEqual(len(content), 3) # Includes empty line at the end of the file\n test_data = [\n b\"Date,report name,Campaign,topic__title,assets total,assets published,\"\n b\"assets not started,assets in progress,assets waiting review,\"\n b\"assets completed,assets unpublished,assets started,items published,\"\n b\"items unpublished,projects published,projects unpublished,\"\n b\"anonymous transcriptions,transcriptions saved,daily review actions,\"\n b\"distinct tags,tag uses,campaigns published,campaigns unpublished,\"\n b\"users registered,users activated,registered contributors,\"\n b\"daily active users\",\n b\"%s,,,,,,,,,,,,,,,,,,,,,,,,,,\" % str.encode(mocked_datetime_formatted),\n b\"\",\n ]\n self.assertEqual(content, test_data)\n\n def test_admin_retired_site_report_view(self):\n self.login_user(is_staff=True, is_superuser=True)\n\n response = self.client.get(reverse(\"admin:retired-site-report\"))\n self.assertEqual(response.status_code, 200)\n content = self.get_streaming_content(response).split(b\"\\r\\n\")\n self.assertEqual(len(content), 3) # Includes empty line at the end of the file\n test_data = [\n b\"Date,report name,Campaign,topic__title,assets total,assets published,\"\n b\"assets not started,assets in progress,assets waiting review,\"\n b\"assets completed,assets unpublished,assets started,items published,\"\n b\"items unpublished,projects published,projects unpublished,\"\n b\"anonymous transcriptions,transcriptions saved,daily review actions,\"\n b\"distinct tags,tag uses,campaigns published,campaigns unpublished,\"\n b\"users registered,users activated,registered contributors,\"\n b\"daily active users\",\n b\",RETIRED TOTAL,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\",\n b\"\",\n ]\n self.assertEqual(content, test_data)\n\n\nclass TestAdminBulkImportView(CreateTestUsers, TestCase):\n def setUp(self):\n self.login_user(is_staff=True, is_superuser=True)\n self.path = reverse(\"admin:bulk-import\")\n self.campaign_title = \"Test Campaign\"\n self.campaign_short_description = \"Short description\"\n self.campaign_long_description = \"Long description\"\n self.campaign_slug = \"test-campaign\"\n self.project_slug = \"test-project\"\n self.project_title = \"Test Project\"\n self.project_description = \"Project description\"\n self.url = \"http://example.com\"\n self.spreadsheet_data = {\n \"Campaign\": self.campaign_title,\n \"Campaign Short Description\": self.campaign_short_description,\n \"Campaign Long Description\": self.campaign_long_description,\n \"Campaign Slug\": self.campaign_slug,\n \"Project Slug\": self.project_slug,\n \"Project\": self.project_title,\n \"Project Description\": self.project_description,\n \"Import URLs\": self.url,\n }\n self.post_data = {\"spreadsheet_file\": BytesIO()}\n\n def test_get(self):\n response = self.client.get(self.path)\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"form\", response.context)\n\n def test_invalid_form(self):\n response = self.client.post(self.path)\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"form\", response.context)\n\n def test_fully_valid_form(self):\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.import_items_into_project_from_url\",\n autospec=True,\n ),\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [self.spreadsheet_data]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(messages[0], f\"Created new campaign {self.campaign_title}\")\n self.assertEqual(messages[1], f\"Created new project {self.project_title}\")\n self.assertEqual(\n messages[2],\n f\"Queued {self.campaign_title} {self.project_title} \"\n f\"import for {self.url}\",\n )\n\n campaign = Campaign.objects.get()\n self.assertEqual(campaign.title, self.campaign_title)\n self.assertEqual(campaign.slug, self.campaign_slug)\n self.assertEqual(campaign.description, self.campaign_long_description)\n self.assertEqual(\n campaign.short_description, self.campaign_short_description\n )\n\n project = Project.objects.get()\n self.assertEqual(project.title, self.project_title)\n self.assertEqual(project.slug, self.project_slug)\n self.assertEqual(project.description, self.project_description)\n\n # Submit it again to test that it doesn't re-create the campaign or project\n response = self.client.post(self.path, data=self.post_data)\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(\n messages[0],\n f\"Reusing campaign {self.campaign_title} without modification\",\n )\n self.assertEqual(\n messages[1],\n f\"Reusing project {self.project_title} without modification\",\n )\n self.assertEqual(\n messages[2],\n f\"Queued {self.campaign_title} {self.project_title} \"\n f\"import for {self.url}\",\n )\n self.assertEqual(1, Campaign.objects.count())\n self.assertEqual(1, Project.objects.count())\n\n def test_missing_field(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n del spreadsheet_data[\"Campaign\"]\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.import_items_into_project_from_url\",\n autospec=True,\n ),\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 1)\n self.assertEqual(\n str(messages[0]), \"Skipping row 0: missing fields ['Campaign']\"\n )\n\n def test_empty_field(self):\n # Only three fields require values: Campaign, Projet and Import URLs.\n # Other fields must be present but can be empty.\n # This tests that blank value check\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.import_items_into_project_from_url\",\n autospec=True,\n ),\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n\n # Test one empty field\n spreadsheet_data[\"Campaign\"] = \"\"\n slurp_mock.return_value = [spreadsheet_data]\n\n response = self.client.post(self.path, data=self.post_data)\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 1)\n self.assertEqual(\n messages[0],\n \"Skipping row 0: at least one required field \"\n \"(Campaign, Project, Import URLs) is empty\",\n )\n\n def test_all_empty_fields(self):\n # If all values in a spreadsheet row are empty, the row is skipped silently\n spreadsheet_data = {key: \"\" for key in self.spreadsheet_data.keys()}\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.import_items_into_project_from_url\",\n autospec=True,\n ),\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 0)\n\n def test_empty_campaign_slug(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Campaign Slug\"] = \"\"\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.import_items_into_project_from_url\",\n autospec=True,\n ),\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(messages[0], f\"Created new campaign {self.campaign_title}\")\n self.assertEqual(messages[1], f\"Created new project {self.project_title}\")\n self.assertEqual(\n messages[2],\n f\"Queued {self.campaign_title} {self.project_title} import for {self.url}\",\n )\n\n # Since the provided campaign slug was blank, it should slugify the Campaign\n # field instead\n campaign = Campaign.objects.get()\n self.assertEqual(\n campaign.slug, slugify(self.campaign_title, allow_unicode=True)\n )\n\n def test_bad_campaign_slug(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Campaign Slug\"] = \"bad#slug@\"\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.import_items_into_project_from_url\",\n autospec=True,\n ),\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 2)\n self.assertEqual(messages[0], \"Campaign slug doesn't match pattern.\")\n self.assertEqual(\n messages[1],\n \"Unable to create campaign Test Campaign: {'slug': \"\n \"['Enter a valid βslugβ consisting of Unicode letters, \"\n \"numbers, underscores, or hyphens.']}\",\n )\n\n def test_empty_project_slug(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Project Slug\"] = \"\"\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.import_items_into_project_from_url\",\n autospec=True,\n ),\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(messages[0], f\"Created new campaign {self.campaign_title}\")\n self.assertEqual(messages[1], f\"Created new project {self.project_title}\")\n self.assertEqual(\n messages[2],\n f\"Queued {self.campaign_title} {self.project_title} import for {self.url}\",\n )\n\n # Since the provided project slug was blank, it should slugify the project\n # field instead\n project = Project.objects.get()\n self.assertEqual(project.slug, slugify(self.project_title, allow_unicode=True))\n\n def test_bad_project_slug(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Project Slug\"] = \"bad#slug@\"\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.import_items_into_project_from_url\",\n autospec=True,\n ),\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(messages[0], f\"Created new campaign {self.campaign_title}\")\n self.assertEqual(messages[1], \"Project slug doesn't match pattern.\")\n self.assertEqual(\n messages[2],\n \"Unable to create project Test Project: {'slug': \"\n \"['Enter a valid βslugβ consisting of Unicode letters, \"\n \"numbers, underscores, or hyphens.']}\",\n )\n\n def test_bad_url(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Import URLs\"] = bad_url = \"ftp://example.com\"\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.import_items_into_project_from_url\",\n autospec=True,\n ),\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(messages[0], f\"Created new campaign {self.campaign_title}\")\n self.assertEqual(messages[1], f\"Created new project {self.project_title}\")\n self.assertEqual(messages[2], f\"Skipping unrecognized URL value: {bad_url}\")\n\n def test_import_task_exception(self):\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.import_items_into_project_from_url\",\n autospec=True,\n ) as import_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [self.spreadsheet_data]\n import_mock.side_effect = Exception(\"Test Exception\")\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(messages[0], f\"Created new campaign {self.campaign_title}\")\n self.assertEqual(messages[1], f\"Created new project {self.project_title}\")\n self.assertEqual(\n messages[2],\n f\"Unhandled error attempting to import {self.url}: Test Exception\",\n )\n\n\nclass TestAdminBulkChangeAssetStatus(CreateTestUsers, TestCase):\n def setUp(self):\n item = create_item()\n self.assets = [\n create_asset(item=item),\n create_asset(item=item, slug=\"test-asset-2\"),\n create_asset(item=item, slug=\"test-asset-3\"),\n create_asset(item=item, slug=\"test-asset-4\"),\n ]\n # Seed with existing transcriptions\n self.accepted_transcription = create_transcription(\n asset=self.assets[0], accepted=now()\n )\n self.rejected_transcription = create_transcription(\n asset=self.assets[1], rejected=now()\n )\n self.submitted_transcription = create_transcription(asset=self.assets[2])\n self.accepted_transcription2 = create_transcription(\n asset=self.assets[3], accepted=now()\n )\n anon = get_anonymous_user()\n self.spreadsheet_data = [\n {\n \"asset__slug\": self.assets[0].slug,\n \"New Status\": \"submitted\",\n \"user\": anon.id,\n },\n {\n \"asset__slug\": self.assets[1].slug,\n \"New Status\": \"completed\",\n \"user\": anon.id,\n },\n {\n \"asset__slug\": self.assets[2].slug,\n \"New Status\": \"completed\",\n \"user\": anon.id,\n },\n {\n \"asset__slug\": self.assets[3].slug,\n \"New Status\": \"in_progress\",\n \"user\": anon.id,\n },\n ]\n\n def test_admin_bulk_change_asset_status(self):\n self.login_user(is_staff=True, is_superuser=True)\n\n fake_file = SimpleUploadedFile(\n \"test.xlsx\", b\"x\", content_type=\"application/vnd.ms-excel\"\n )\n post_data = {\"spreadsheet_file\": fake_file}\n\n with mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock:\n slurp_mock.return_value = self.spreadsheet_data\n\n path = reverse(\"admin:bulk-change\")\n response = self.client.post(path, data=post_data)\n for asset in self.assets:\n asset.refresh_from_db()\n self.assertEqual(response.status_code, 200)\n slurp_mock.assert_called()\n self.assertEqual(\n self.assets[0].transcription_status,\n TranscriptionStatus.SUBMITTED,\n )\n self.assertEqual(\n self.assets[1].transcription_status,\n TranscriptionStatus.COMPLETED,\n )\n self.assertEqual(\n self.assets[2].transcription_status,\n TranscriptionStatus.COMPLETED,\n )\n self.assertEqual(\n self.assets[3].transcription_status,\n TranscriptionStatus.IN_PROGRESS,\n )\n\n\nclass TestAdminBulkImportReview(CreateTestUsers, TestCase):\n def setUp(self):\n self.login_user(is_staff=True, is_superuser=True)\n self.path = reverse(\"admin:bulk-review\")\n self.campaign_title = \"Test Campaign\"\n self.campaign_short_description = \"Short description\"\n self.campaign_long_description = \"Long description\"\n self.campaign_slug = \"test-campaign\"\n self.project_slug = \"test-project\"\n self.project_title = \"Test Project\"\n self.project_description = \"Project description\"\n self.url = \"http://example.com\"\n self.spreadsheet_data = {\n \"Campaign\": self.campaign_title,\n \"Campaign Short Description\": self.campaign_short_description,\n \"Campaign Long Description\": self.campaign_long_description,\n \"Campaign Slug\": self.campaign_slug,\n \"Project Slug\": self.project_slug,\n \"Project\": self.project_title,\n \"Project Description\": self.project_description,\n \"Import URLs\": self.url,\n }\n self.post_data = {\"spreadsheet_file\": BytesIO()}\n\n def test_get(self):\n response = self.client.get(self.path)\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"form\", response.context)\n\n def test_invalid_form(self):\n response = self.client.post(self.path)\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"form\", response.context)\n\n def test_fully_valid_form(self):\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.fetch_all_urls\",\n autospec=True,\n ) as fetch_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [self.spreadsheet_data]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(messages[0], \"Fetch test message\")\n self.assertEqual(messages[1], \"Total Asset Count:1\")\n self.assertEqual(messages[2], \"All Processes Completed\")\n\n def test_missing_field(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n del spreadsheet_data[\"Campaign\"]\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.fetch_all_urls\",\n autospec=True,\n ) as fetch_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 4)\n self.assertEqual(messages[0], \"Skipping row 0: missing fields ['Campaign']\")\n self.assertEqual(messages[1], \"Fetch test message\")\n self.assertEqual(messages[2], \"Total Asset Count:1\")\n self.assertEqual(messages[3], \"All Processes Completed\")\n\n def test_empty_field(self):\n # Only three fields require values: Campaign, Projet and Import URLs.\n # Other fields must be present but can be empty.\n # This tests that blank value check\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.fetch_all_urls\",\n autospec=True,\n ) as fetch_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n # Test one empty field\n spreadsheet_data[\"Campaign\"] = \"\"\n slurp_mock.return_value = [spreadsheet_data]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n\n response = self.client.post(self.path, data=self.post_data)\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 4)\n self.assertEqual(\n messages[0],\n \"Skipping row 0: at least one required field \"\n \"(Campaign, Project, Import URLs) is empty\",\n )\n self.assertEqual(messages[1], \"Fetch test message\")\n self.assertEqual(messages[2], \"Total Asset Count:1\")\n self.assertEqual(messages[3], \"All Processes Completed\")\n\n def test_all_empty_fields(self):\n # If all values in a spreadsheet row are empty, the row is skipped silently\n spreadsheet_data = {key: \"\" for key in self.spreadsheet_data.keys()}\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.fetch_all_urls\",\n autospec=True,\n ) as fetch_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(messages[0], \"Fetch test message\")\n self.assertEqual(messages[1], \"Total Asset Count:1\")\n self.assertEqual(messages[2], \"All Processes Completed\")\n\n def test_empty_campaign_slug(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Campaign Slug\"] = \"\"\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.fetch_all_urls\",\n autospec=True,\n ) as fetch_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(messages[0], \"Fetch test message\")\n self.assertEqual(messages[1], \"Total Asset Count:1\")\n self.assertEqual(messages[2], \"All Processes Completed\")\n\n def test_bad_campaign_slug(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Campaign Slug\"] = \"bad#slug@\"\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.fetch_all_urls\",\n autospec=True,\n ) as fetch_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 4)\n self.assertEqual(messages[0], \"Campaign slug doesn't match pattern.\")\n self.assertEqual(messages[1], \"Fetch test message\")\n self.assertEqual(messages[2], \"Total Asset Count:1\")\n self.assertEqual(messages[3], \"All Processes Completed\")\n\n def test_empty_project_slug(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Project Slug\"] = \"\"\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.fetch_all_urls\",\n autospec=True,\n ) as fetch_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 3)\n self.assertEqual(messages[0], \"Fetch test message\")\n self.assertEqual(messages[1], \"Total Asset Count:1\")\n self.assertEqual(messages[2], \"All Processes Completed\")\n\n def test_bad_project_slug(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Project Slug\"] = \"bad#slug@\"\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.fetch_all_urls\",\n autospec=True,\n ) as fetch_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 4)\n self.assertEqual(messages[0], \"Project slug doesn't match pattern.\")\n self.assertEqual(messages[1], \"Fetch test message\")\n self.assertEqual(messages[2], \"Total Asset Count:1\")\n self.assertEqual(messages[3], \"All Processes Completed\")\n\n def test_bad_url(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Import URLs\"] = bad_url = \"ftp://example.com\"\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.fetch_all_urls\",\n autospec=True,\n ) as fetch_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 4)\n self.assertEqual(messages[0], f\"Skipping unrecognized URL value: {bad_url}\")\n self.assertEqual(messages[1], \"Fetch test message\")\n self.assertEqual(messages[2], \"Total Asset Count:1\")\n self.assertEqual(messages[3], \"All Processes Completed\")\n\n def test_large_number_urls(self):\n spreadsheet_data = copy.copy(self.spreadsheet_data)\n spreadsheet_data[\"Import URLs\"] = \" \".join([self.url for i in range(51)])\n\n with (\n mock.patch(\n \"concordia.admin.views.AdminProjectBulkImportForm\", autospec=True\n ) as form_mock,\n mock.patch(\n \"concordia.admin.views.slurp_excel\", autospec=True\n ) as slurp_mock,\n mock.patch(\n \"concordia.admin.views.fetch_all_urls\",\n autospec=True,\n ) as fetch_mock,\n ):\n form_mock.return_value.is_valid.return_value = True\n form_mock.return_value.cleaned_data = {}\n slurp_mock.return_value = [spreadsheet_data]\n fetch_mock.return_value = [[\"Fetch test message\"], 1]\n\n response = self.client.post(self.path, data=self.post_data)\n\n self.assertEqual(response.status_code, 200)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(len(messages), 4)\n self.assertEqual(messages[0], \"Fetch test message\")\n self.assertEqual(messages[1], \"Fetch test message\")\n # This count is weird because we mock the fetch_all_urls function\n self.assertEqual(messages[2], \"Total Asset Count:2\")\n self.assertEqual(messages[3], \"All Processes Completed\")\n\n\nclass TestCeleryTaskReview(CreateTestUsers, TestCase):\n def setUp(self):\n # We don't set up our data here because we want to test\n # both with and without data\n self.login_user(is_staff=True, is_superuser=True)\n self.path = reverse(\"admin:celery-review\")\n\n def add_campaigns(self):\n self.add_active_campaigns()\n self.add_completed_campaigns()\n self.add_retired_campaigns()\n\n def add_active_campaigns(self):\n self.campaign1 = create_campaign(\n slug=\"test-active-campaign-1\", title=\"Test Active Campaign 1\"\n )\n self.campaign2 = create_campaign(\n slug=\"test-active-campaign-2\", title=\"Test Active Campaign 2\"\n )\n\n def add_completed_campaigns(self):\n self.completed_campaign1 = create_campaign(\n slug=\"test-completed-campaign-1\",\n title=\"Test Completed Campaign 1\",\n status=Campaign.Status.COMPLETED,\n )\n self.completed_campaign2 = create_campaign(\n slug=\"test-completed-campaign-2\",\n title=\"Test Completed Campaign 1\",\n status=Campaign.Status.COMPLETED,\n )\n\n def add_retired_campaigns(self):\n self.retired_campaign1 = create_campaign(\n slug=\"test-retired-campaign-1\",\n title=\"Test Retired Campaign 1\",\n status=Campaign.Status.RETIRED,\n )\n self.retired_campaign2 = create_campaign(\n slug=\"test-retired-campaign-2\",\n title=\"Test Retired Campaign 1\",\n status=Campaign.Status.RETIRED,\n )\n\n def add_projects(self):\n # Active campaign 1, three projects\n create_project(\n campaign=self.campaign1,\n slug=\"campaign1-project-1\",\n title=\"Campaign 1 Project 1\",\n )\n create_project(\n campaign=self.campaign1,\n slug=\"campaign1-project-2\",\n title=\"Campaign 1 Project 2\",\n )\n create_project(\n campaign=self.campaign1,\n slug=\"campaign1-project-3\",\n title=\"Campaign 1 Project 3\",\n )\n\n # Active campaign 2, two projects\n create_project(\n campaign=self.campaign2,\n slug=\"campaign1-project-1\",\n title=\"Campaign 2 Project 1\",\n )\n create_project(\n campaign=self.campaign2,\n slug=\"campaign1-project-2\",\n title=\"Campaign 2 Project 2\",\n )\n\n # Completed campaign 1, two projects\n create_project(\n campaign=self.completed_campaign1,\n slug=\"completed-campaign1-project-1\",\n title=\"Completed Campaign 1 Project 1\",\n )\n create_project(\n campaign=self.completed_campaign1,\n slug=\"completed-campaign1-project-2\",\n title=\"Completed Campaign 1 Project 2\",\n )\n\n # Completed campaign 2, one project\n create_project(\n campaign=self.completed_campaign2,\n slug=\"completed-campaign2-project-1\",\n title=\"Completed Campaign 1 Project 1\",\n )\n\n # We don't create any for retired campaigns since the campaigns\n # are only created to make sure the view ignores them\n\n def add_tasks(self, campaign):\n data = []\n for project in campaign.project_set.all():\n import_asset = create_import_asset(1, project=project)\n item = import_asset.import_item\n import_job = item.job\n create_import_asset(\n 2,\n import_item=item,\n import_job=import_job,\n project=project,\n last_started=timezone.now(),\n )\n create_import_asset(\n 3,\n import_item=item,\n import_job=import_job,\n project=project,\n failed=timezone.now(),\n last_started=timezone.now(),\n )\n create_import_asset(\n 4,\n import_item=item,\n import_job=import_job,\n project=project,\n completed=timezone.now(),\n last_started=timezone.now(),\n )\n data.append(\n {\n \"title\": project.title,\n \"id\": project.id,\n \"campaign_id\": str(campaign.id),\n \"successful\": 1,\n \"incomplete\": 1,\n \"unstarted\": 1,\n \"failure\": 1,\n }\n )\n return data\n\n def test_empty_dashboard(self):\n response = self.client.get(self.path)\n context = response.context\n\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"campaigns\", context)\n campaigns = list(context[\"campaigns\"])\n self.assertEqual(campaigns, [])\n\n def test_dashboard(self):\n self.add_active_campaigns()\n response = self.client.get(self.path)\n context = response.context\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"campaigns\", context)\n self.assertIn(self.campaign1, context[\"campaigns\"])\n self.assertIn(self.campaign2, context[\"campaigns\"])\n\n self.add_completed_campaigns()\n response = self.client.get(self.path)\n context = response.context\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"campaigns\", context)\n campaigns = list(context[\"campaigns\"])\n self.assertIn(self.campaign1, campaigns)\n self.assertIn(self.campaign2, campaigns)\n self.assertIn(self.completed_campaign1, campaigns)\n self.assertIn(self.completed_campaign2, campaigns)\n\n self.add_retired_campaigns()\n response = self.client.get(self.path)\n context = response.context\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"campaigns\", context)\n campaigns = list(context[\"campaigns\"])\n self.assertIn(self.campaign1, campaigns)\n self.assertIn(self.campaign2, campaigns)\n self.assertIn(self.completed_campaign1, campaigns)\n self.assertIn(self.completed_campaign2, campaigns)\n self.assertNotIn(self.retired_campaign1, campaigns)\n self.assertNotIn(self.retired_campaign2, campaigns)\n\n def test_campaign_dashboard(self):\n self.add_campaigns()\n self.add_projects()\n\n data = self.add_tasks(self.campaign1)\n response = self.client.get(self.path, {\"id\": self.campaign1.id})\n context = response.context\n self.assertIn(\"campaigns\", context)\n self.assertEqual(context[\"campaigns\"], [])\n self.assertIn(\"totalassets\", context)\n self.assertEqual(context[\"totalassets\"], 12)\n self.assertIn(\"projects\", context)\n self.assertEqual(context[\"projects\"], data)\n\n data = self.add_tasks(self.campaign2)\n response = self.client.get(self.path, {\"id\": self.campaign2.id})\n context = response.context\n self.assertIn(\"campaigns\", context)\n self.assertEqual(context[\"campaigns\"], [])\n self.assertIn(\"totalassets\", context)\n self.assertEqual(context[\"totalassets\"], 8)\n self.assertIn(\"projects\", context)\n self.assertEqual(context[\"projects\"], data)\n\n data = self.add_tasks(self.completed_campaign1)\n response = self.client.get(self.path, {\"id\": self.completed_campaign1.id})\n context = response.context\n self.assertIn(\"campaigns\", context)\n self.assertEqual(context[\"campaigns\"], [])\n self.assertIn(\"totalassets\", context)\n self.assertEqual(context[\"totalassets\"], 8)\n self.assertIn(\"projects\", context)\n self.assertEqual(context[\"projects\"], data)\n\n data = self.add_tasks(self.completed_campaign2)\n response = self.client.get(self.path, {\"id\": self.completed_campaign2.id})\n context = response.context\n self.assertIn(\"campaigns\", context)\n self.assertEqual(context[\"campaigns\"], [])\n self.assertIn(\"totalassets\", context)\n self.assertEqual(context[\"totalassets\"], 4)\n self.assertIn(\"projects\", context)\n self.assertEqual(context[\"projects\"], data)\n\n\nclass TestSerializedObjectView(TestCase):\n def setUp(self):\n self.card = create_card()\n # Every test needs access to the request factory.\n self.factory = RequestFactory()\n\n def test_exists(self):\n request = self.factory.get(\n \"/admin/card/\",\n {\"model_name\": \"Card\", \"object_id\": self.card.id, \"field_name\": \"title\"},\n )\n response = SerializedObjectView.as_view()(request)\n self.assertEqual(response.status_code, 200)\n self.assertEqual(json.loads(response.content)[\"title\"], self.card.title)\n\n def test_dne(self):\n request = self.factory.get(\n \"/admin/card/\",\n {\"model_name\": \"Card\", \"object_id\": 3, \"field_name\": \"title\"},\n )\n response = SerializedObjectView.as_view()(request)\n self.assertEqual(response.status_code, HTTPStatus.NOT_FOUND)\n self.assertJSONEqual(response.content, {\"status\": \"false\"})\n\n\ndef mock_cache(object_to_patch):\n def decorator(cls):\n # Decorator to mock `django.core.cache.caches`.\n # Passes the mock cache and caches_mock to each\n # test method as additional arguments.\n # We have to write this as a custom decorator\n # in order to not have to create these mocks in\n # each invidivual test method, since we need to override\n # __getitem__ on the caches mock\n\n # We need to create a helper function so each method\n # gets a unique wrapper and mocks\n def create_wrapper(attr):\n @wraps(attr)\n def wrapper(self, *args, **kwargs):\n with mock.patch(object_to_patch) as caches_mock:\n cache_mock = mock.MagicMock()\n caches_mock.__getitem__.return_value = cache_mock\n return attr(self, caches_mock, cache_mock, *args, **kwargs)\n\n return wrapper\n\n # Wrap each test method to include the mocks as arguments\n for attr_name in dir(cls):\n attr = getattr(cls, attr_name)\n if callable(attr) and attr_name.startswith(\"test_\"):\n setattr(cls, attr_name, create_wrapper(attr))\n\n return cls\n\n return decorator\n\n\n@mock_cache(\"concordia.admin.views.caches\")\nclass TestClearCacheView(CreateTestUsers, TestCase):\n def setUp(self):\n self.login_user(is_staff=True, is_superuser=True)\n self.path = reverse(\"admin:clear-cache\")\n\n def test_get(self, caches_mock, cache_mock):\n response = self.client.get(self.path)\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"form\", response.context)\n self.assertFalse(caches_mock.__getitem__.called)\n self.assertFalse(cache_mock.clear.called)\n\n def test_invalid_form(self, caches_mock, cache_mock):\n response = self.client.post(self.path)\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"form\", response.context)\n self.assertFalse(caches_mock.__getitem__.called)\n self.assertFalse(cache_mock.clear.called)\n\n def test_valid_form(self, caches_mock, cache_mock):\n response = self.client.post(self.path, {\"cache_name\": \"view_cache\"})\n self.assertEqual(response.status_code, 302)\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(messages[0], \"Successfully cleared 'view_cache' cache\")\n self.assertTrue(caches_mock.__getitem__.called)\n self.assertTrue(cache_mock.clear.called)\n\n def test_form_with_invalid_data(self, caches_mock, cache_mock):\n response = self.client.post(self.path, {\"cache_name\": \"default\"})\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"form\", response.context)\n self.assertFalse(caches_mock.__getitem__.called)\n self.assertFalse(cache_mock.clear.called)\n\n def test_exception(self, caches_mock, cache_mock):\n caches_mock.__getitem__.side_effect = Exception(\"Test Exception\")\n response = self.client.post(self.path, {\"cache_name\": \"view_cache\"})\n messages = [str(message) for message in get_messages(response.wsgi_request)]\n self.assertEqual(\n messages[0],\n \"Couldn't clear cache 'view_cache', something went wrong. \"\n \"Received error: Test Exception\",\n )\n"
},
{
"path": "concordia/tests/test_api_views.py",
"content": "from datetime import date\nfrom unittest import mock\nfrom urllib.parse import urlparse\n\nfrom django.test import TestCase, override_settings\nfrom django.urls import reverse\nfrom django.utils.timezone import now\n\nfrom concordia import api_views\nfrom concordia.models import (\n Asset,\n Campaign,\n Item,\n Project,\n Topic,\n Transcription,\n User,\n)\nfrom concordia.utils import get_anonymous_user\n\nfrom .utils import (\n JSONAssertMixin,\n create_asset,\n create_item,\n create_project,\n create_topic,\n)\n\n\nclass URLAwareEncoderTest(TestCase):\n def test_default(self):\n encoder = api_views.URLAwareEncoder()\n self.assertEqual(encoder.default(None), None)\n\n obj = mock.Mock(spec=[\"url\"])\n self.assertEqual(encoder.default(obj), obj.url)\n\n obj = mock.Mock(spec=[\"get_absolute_url\"])\n self.assertEqual(encoder.default(obj), obj.get_absolute_url())\n\n # Test non-model object\n obj = date.today()\n self.assertEqual(encoder.default(obj), date.today().isoformat())\n\n\nclass APIViewMixinTest(TestCase):\n def setUp(self):\n self.mixin = api_views.APIViewMixin()\n\n def test_serialize_conctext(self):\n context = {\"test-key\": \"test-value\"}\n self.assertEqual(self.mixin.serialize_context(context), context)\n\n @mock.patch(\"concordia.api_views.model_to_dict\")\n def test_serialize_object(self, mtd_mock):\n return_data = {\"test-key\": \"test-value\"}\n mtd_mock.return_value = return_data\n\n obj = mock.Mock(spec=[\"get_absolute_url\"])\n data = self.mixin.serialize_object(obj)\n\n self.assertEqual(data, return_data | {\"url\": obj.get_absolute_url()})\n\n obj = mock.Mock(spec=[])\n data = self.mixin.serialize_object(obj)\n\n self.assertEqual(data, return_data)\n\n\n@mock.patch(\"concordia.api_views.time\")\nclass APIListViewTest(TestCase):\n def test_serialize_context(self, time_mock):\n time_mock.return_value = \"test-time\"\n view = api_views.APIListView()\n context = {\"object_list\": []}\n\n data = view.serialize_context(context)\n self.assertEqual(data, {\"objects\": [], \"sent\": \"test-time\"})\n\n\n@override_settings(RATELIMIT_ENABLE=False)\nclass ConcordiaViewTests(JSONAssertMixin, TestCase):\n @classmethod\n def setUpTestData(cls):\n cls.anon_user = get_anonymous_user()\n\n cls.reviewer = User.objects.create_user(\n username=\"reviewer\", email=\"tester@example.com\"\n )\n\n # clear data from other tests\n Project.objects.all().delete()\n Topic.objects.all().delete()\n\n cls.test_project = create_project()\n\n cls.test_topic = create_topic(project=cls.test_project)\n\n cls.items = [\n create_item(\n item_id=f\"item_{i}\",\n title=f\"Item {i}\",\n project=cls.test_project,\n do_save=False,\n )\n for i in range(0, 3)\n ]\n Item.objects.bulk_create(cls.items)\n\n cls.assets = []\n for item in cls.items:\n cls.assets.append(\n create_asset(\n title=f\"Thumbnail URL test for {item.id}\",\n item=item,\n download_url=\"http://tile.loc.gov/image-services/iiif/\"\n \"service:music:mussuffrage:mussuffrage-100183:mussuffrage-100183.0001/\"\n \"full/pct:100/0/default.jpg\",\n do_save=False,\n )\n )\n for i in range(0, 15):\n cls.assets.append(\n create_asset(title=f\"{item.id} β {i}\", item=item, do_save=False)\n )\n Asset.objects.bulk_create(cls.assets)\n\n cls.transcriptions = []\n for asset in cls.assets:\n last_t = None\n\n for n in range(0, 3):\n cls.transcriptions.append(\n Transcription(\n asset=asset,\n supersedes=last_t,\n text=f\"{asset} β {n}\",\n user=cls.anon_user,\n )\n )\n\n Transcription.objects.bulk_create(cls.transcriptions)\n\n submitted_t = cls.transcriptions[-1]\n submitted_t.submitted = now()\n submitted_t.full_clean()\n submitted_t.save()\n\n def get_api_response(self, url, **request_args):\n \"\"\"\n This issues a call to one of our API views and confirms that the\n response follows our basic conventions of returning a valid JSON\n response\n \"\"\"\n\n qs = {\"format\": \"json\"}\n if request_args is not None:\n qs.update(request_args)\n\n resp = self.client.get(url, qs)\n data = self.assertValidJSON(resp)\n return resp, data\n\n def get_api_list_response(self, url, page_size=10, **request_args):\n \"\"\"\n This issues a call to one of our API views and confirms that the\n response follows our basic conventions of returning a top level object\n with membersβobjectsβ (list) and βpaginationβ (object).\n \"\"\"\n\n qs = {\"per_page\": page_size}\n if request_args is not None:\n qs.update(request_args)\n\n resp, data = self.get_api_response(url, **qs)\n\n self.assertIn(\"objects\", data)\n self.assertIn(\"pagination\", data)\n\n object_count = len(data[\"objects\"])\n self.assertLessEqual(object_count, page_size)\n\n self.assertAbsoluteURLs(data[\"objects\"])\n self.assertAbsoluteURLs(data[\"pagination\"])\n\n return resp, data\n\n def assertAbsoluteUrl(self, url, allow_none=True):\n \"\"\"Require a URL to either be None or an absolute URL\"\"\"\n\n if url is None and allow_none:\n return\n\n parsed = urlparse(url)\n self.assertIn(\n parsed.scheme, [\"http\", \"https\"], msg=f\"Expected {url} to have HTTP scheme\"\n )\n\n self.assertTrue(parsed.netloc)\n\n def assertAbsoluteURLs(self, data):\n if isinstance(data, dict):\n for k, v in data.items():\n if k.endswith(\"url\"):\n self.assertAbsoluteUrl(v)\n elif isinstance(v, (dict, list)):\n self.assertAbsoluteURLs(v)\n elif isinstance(data, list):\n for i in data:\n self.assertAbsoluteURLs(i)\n else:\n raise TypeError(\n \"assertAbsoluteURLs must be called with a dictionary or list\"\n )\n\n def assertAssetStatuses(self, asset_list, expected_statuses):\n asset_pks = [i[\"id\"] for i in asset_list]\n\n self.assertQuerySetEqual(\n Asset.objects.filter(pk__in=asset_pks).exclude(\n transcription_status__in=expected_statuses\n ),\n [],\n )\n\n def assertAssetsHaveLatestTranscriptions(self, asset_list):\n asset_pks = {i[\"id\"]: i for i in asset_list}\n\n for asset in Asset.objects.filter(pk__in=asset_pks.keys()):\n latest_trans = asset.transcription_set.latest(\"pk\")\n\n if latest_trans is None:\n self.assertIsNone(asset_pks[asset.id][\"latest_transcription\"])\n else:\n self.assertDictEqual(\n asset_pks[asset.id][\"latest_transcription\"],\n {\n \"id\": latest_trans.pk,\n \"text\": latest_trans.text,\n \"submitted_by\": latest_trans.user_id,\n },\n )\n\n def test_topic_detail(self):\n resp, data = self.get_api_response(\n reverse(\"topic-detail\", kwargs={\"slug\": self.test_topic.slug})\n )\n\n self.assertIn(\"object\", data)\n self.assertNotIn(\"objects\", data)\n\n serialized_project = data[\"object\"]\n\n self.assertIn(\"id\", serialized_project)\n self.assertIn(\"url\", serialized_project)\n topic = self.test_topic\n self.assertEqual(\n serialized_project,\n serialized_project\n | {\n \"id\": topic.id,\n \"title\": topic.title,\n \"description\": topic.description,\n \"slug\": topic.slug,\n \"thumbnail_image\": topic.thumbnail_image,\n },\n )\n self.assertURLEqual(\n serialized_project[\"url\"], f\"http://testserver{topic.get_absolute_url()}\"\n )\n\n def test_campaign_list(self):\n resp, data = self.get_api_list_response(reverse(\"transcriptions:campaign-list\"))\n\n self.assertGreater(len(data[\"objects\"]), 0)\n\n test_campaigns = {\n i[\"id\"]: i\n for i in Campaign.objects.published().values(\n \"id\", \"title\", \"description\", \"short_description\", \"slug\"\n )\n }\n\n for obj in data[\"objects\"]:\n self.assertIn(\"id\", obj)\n self.assertIn(\"url\", obj)\n self.assertEqual(obj, obj | test_campaigns[obj[\"id\"]])\n\n def test_campaign_detail(self):\n resp, data = self.get_api_response(\n reverse(\n \"transcriptions:campaign-detail\",\n kwargs={\"slug\": self.test_project.campaign.slug},\n )\n )\n\n self.assertIn(\"object\", data)\n self.assertNotIn(\"objects\", data)\n\n serialized_project = data[\"object\"]\n\n self.assertIn(\"id\", serialized_project)\n self.assertIn(\"url\", serialized_project)\n campaign = self.test_project.campaign\n self.assertEqual(\n serialized_project,\n serialized_project\n | {\n \"id\": campaign.id,\n \"title\": campaign.title,\n \"description\": campaign.description,\n \"slug\": campaign.slug,\n \"metadata\": campaign.metadata,\n \"thumbnail_image\": campaign.thumbnail_image,\n },\n )\n self.assertURLEqual(\n serialized_project[\"url\"], f\"http://testserver{campaign.get_absolute_url()}\"\n )\n\n def test_project_detail(self):\n project = self.test_project\n\n resp, data = self.get_api_list_response(project.get_absolute_url())\n\n # Until we clean up the project view code, projects have two key\n # elements: objects lists the children (i.e. items) and the project\n # itself is in a second top-level βprojectβ object:\n self.assertIn(\"objects\", data)\n self.assertIn(\"project\", data)\n self.assertNotIn(\"object\", data)\n\n serialized_project = data[\"project\"]\n\n self.assertIn(\"id\", serialized_project)\n self.assertIn(\"url\", serialized_project)\n\n self.assertURLEqual(\n serialized_project[\"url\"], f\"http://testserver{project.get_absolute_url()}\"\n )\n self.assertEqual(\n serialized_project,\n serialized_project\n | {\n \"description\": project.description,\n \"id\": project.id,\n \"metadata\": project.metadata,\n \"slug\": project.slug,\n \"thumbnail_image\": project.thumbnail_image,\n \"title\": project.title,\n },\n )\n\n for obj in data[\"objects\"]:\n self.assertIn(\"description\", obj)\n self.assertIn(\"item_id\", obj)\n self.assertIn(\"item_url\", obj)\n self.assertIn(\"metadata\", obj)\n self.assertIn(\"thumbnail_url\", obj)\n self.assertIn(\"title\", obj)\n self.assertIn(\"url\", obj)\n\n def test_item_detail(self):\n item = self.test_project.item_set.first()\n resp, data = self.get_api_list_response(item.get_absolute_url())\n\n # Until we clean up the project view code, projects have two key\n # elements: objects lists the children (i.e. items) and the project\n # itself is in a second top-level βprojectβ object:\n self.assertIn(\"objects\", data)\n self.assertIn(\"item\", data)\n self.assertNotIn(\"object\", data)\n\n serialized_item = data[\"item\"]\n\n self.assertIn(\"id\", serialized_item)\n self.assertIn(\"url\", serialized_item)\n self.assertIn(\"thumbnail_url\", serialized_item)\n\n self.assertURLEqual(\n serialized_item[\"url\"], f\"http://testserver{item.get_absolute_url()}\"\n )\n self.assertEqual(\n serialized_item,\n serialized_item\n | {\n \"description\": item.description,\n \"id\": item.id,\n \"item_id\": item.item_id,\n \"metadata\": item.metadata,\n \"title\": item.title,\n },\n )\n\n for obj in data[\"objects\"]:\n self.assertIn(\"description\", obj)\n self.assertIn(\"difficulty\", obj)\n self.assertIn(\"metadata\", obj)\n self.assertIn(\"image_url\", obj)\n self.assertIn(\"thumbnail_url\", obj)\n self.assertIn(\"resource_url\", obj)\n self.assertIn(\"title\", obj)\n self.assertIn(\"slug\", obj)\n self.assertIn(\"url\", obj)\n self.assertIn(\"year\", obj)\n if \"Thumbnail test\" in obj[\"title\"]:\n self.assertIn(\"https\", obj[\"thumbnail_url\"])\n"
},
{
"path": "concordia/tests/test_authentication.py",
"content": "from django.test import RequestFactory, TestCase\n\nfrom concordia.authentication_backends import EmailOrUsernameModelBackend\n\nfrom .utils import CreateTestUsers\n\n\nclass AuthenticationBackendTests(TestCase, CreateTestUsers):\n def test_EmailOrUsernameModelBackend(self):\n backend = EmailOrUsernameModelBackend()\n request_factory = RequestFactory()\n test_user = self.create_user(\"tester\")\n request = request_factory.get(\"/\")\n\n # Fail to authenticate with no information\n user = backend.authenticate(request)\n self.assertEqual(user, None)\n\n # Fail to authenticate with no password, using username\n user = backend.authenticate(request, test_user.username)\n self.assertEqual(user, None)\n\n # Authenticate with correct password, using username\n user = backend.authenticate(request, test_user.username, test_user._password)\n self.assertEqual(user, test_user)\n\n # Fail to authenticate with no password, using email\n user = backend.authenticate(request, test_user.email)\n self.assertEqual(user, None)\n\n # Authenticate with correct password, using email\n user = backend.authenticate(request, test_user.email, test_user._password)\n self.assertEqual(user, test_user)\n\n # Fail to authenticate with incorrect password, using username\n user = backend.authenticate(request, test_user.username, \"bad-password\")\n self.assertEqual(user, None)\n\n # Fail to authenticate with incorrect password, using email\n user = backend.authenticate(request, test_user.email, \"bad-password\")\n self.assertEqual(user, None)\n\n # Same tests, with user with a username\n # the same as the first user's email address\n test_user2 = self.create_user(test_user.email)\n\n # Fail to authenticate with no password, using username\n user = backend.authenticate(request, test_user2.username)\n self.assertEqual(user, None)\n\n # Authenticate with correct password, using username\n user = backend.authenticate(request, test_user2.username, test_user2._password)\n self.assertEqual(user, test_user2)\n\n # Fail to authenticate with no password, using email\n user = backend.authenticate(request, test_user2.email)\n self.assertEqual(user, None)\n\n # Authenticate with correct password, using email\n user = backend.authenticate(request, test_user2.email, test_user2._password)\n self.assertEqual(user, test_user2)\n\n # Fail to authenticate with incorrect password, using username\n user = backend.authenticate(request, test_user2.username, \"bad-password\")\n self.assertEqual(user, None)\n\n # Fail to authenticate with incorrect password, using email\n user = backend.authenticate(request, test_user2.email, \"bad-password\")\n self.assertEqual(user, None)\n"
},
{
"path": "concordia/tests/test_celery.py",
"content": "import tempfile\nfrom types import SimpleNamespace\nfrom unittest import mock\n\nfrom django.test import TestCase\n\nimport concordia.celery as celery_mod\nfrom concordia.celery import import_all_submodules\n\n\nclass ConcordiaCeleryTests(TestCase):\n def test_returns_early_for_non_package(self):\n mock_pkg = SimpleNamespace(__name__=\"not_a_pkg\") # no __path__\n\n with (\n mock.patch.object(\n celery_mod.importlib, \"import_module\", return_value=mock_pkg\n ) as mock_import,\n mock.patch.object(celery_mod.pkgutil, \"walk_packages\") as mock_walk,\n ):\n import_all_submodules(\"not_a_pkg\")\n\n mock_import.assert_called_once_with(\"not_a_pkg\")\n mock_walk.assert_not_called()\n\n def test_imports_all_submodules_for_package(self):\n sub1 = SimpleNamespace(name=\"dummy_pkg.sub1\")\n sub2 = SimpleNamespace(name=\"dummy_pkg.sub2\")\n\n with tempfile.TemporaryDirectory() as td:\n mock_pkg = SimpleNamespace(__name__=\"dummy_pkg\", __path__=[td])\n\n with (\n mock.patch.object(celery_mod.importlib, \"import_module\") as mock_import,\n mock.patch.object(\n celery_mod.pkgutil, \"walk_packages\", return_value=[sub1, sub2]\n ) as mock_walk,\n ):\n\n def side_effect(name):\n if name == \"dummy_pkg\":\n return mock_pkg\n return SimpleNamespace(__name__=name)\n\n mock_import.side_effect = side_effect\n import_all_submodules(\"dummy_pkg\")\n\n mock_walk.assert_called_once()\n args, _kwargs = mock_walk.call_args\n self.assertEqual(args[0], mock_pkg.__path__)\n self.assertEqual(args[1], mock_pkg.__name__ + \".\")\n\n self.assertIn(mock.call(\"dummy_pkg\"), mock_import.mock_calls)\n self.assertIn(mock.call(\"dummy_pkg.sub1\"), mock_import.mock_calls)\n self.assertIn(mock.call(\"dummy_pkg.sub2\"), mock_import.mock_calls)\n\n def test_package_with_no_submodules(self):\n with tempfile.TemporaryDirectory() as td:\n mock_pkg = SimpleNamespace(__name__=\"empty_pkg\", __path__=[td])\n\n with (\n mock.patch.object(celery_mod.importlib, \"import_module\") as mock_import,\n mock.patch.object(\n celery_mod.pkgutil, \"walk_packages\", return_value=[]\n ) as mock_walk,\n ):\n\n mock_import.side_effect = lambda name: (\n mock_pkg if name == \"empty_pkg\" else SimpleNamespace(__name__=name)\n )\n import_all_submodules(\"empty_pkg\")\n\n mock_walk.assert_called_once()\n mock_import.assert_called_once_with(\"empty_pkg\")\n\n def test__load_all_task_modules_invokes_imports(self):\n with mock.patch.object(celery_mod, \"import_all_submodules\") as mock_import_all:\n celery_mod._load_all_task_modules(sender=celery_mod.app)\n\n mock_import_all.assert_has_calls(\n [\n mock.call(\"concordia.tasks\"),\n mock.call(\"importer.tasks\"),\n ],\n any_order=False,\n )\n\n def test_on_after_finalize_signal_triggers_handler(self):\n with mock.patch.object(celery_mod, \"import_all_submodules\") as mock_import_all:\n celery_mod.app.on_after_finalize.send(sender=celery_mod.app)\n\n mock_import_all.assert_has_calls(\n [mock.call(\"concordia.tasks\"), mock.call(\"importer.tasks\")],\n any_order=False,\n )\n self.assertEqual(mock_import_all.call_count, 2)\n"
},
{
"path": "concordia/tests/test_consumers.py",
"content": "from asgiref.sync import sync_to_async\nfrom channels.testing import WebsocketCommunicator\nfrom django.test import RequestFactory, TransactionTestCase\nfrom django.urls import reverse\n\nfrom concordia.consumers import AssetConsumer\nfrom concordia.utils import get_or_create_reservation_token\nfrom concordia.views.ajax import obtain_reservation\n\nfrom .utils import CreateTestUsers, create_asset, create_item, create_transcription\n\n\nclass TestAssetConsumer(CreateTestUsers, TransactionTestCase):\n \"\"\"\n Normally defining communicator would be in setUp\n and communicator.disconnect would be called in tearDown\n Asynchronous code doesn't seem to work well with those methods\n so those lines are in each test.\n \"\"\"\n\n async def test_asset_update(self):\n communicator = WebsocketCommunicator(\n AssetConsumer.as_asgi(), \"ws/asset/asset_updates/\"\n )\n connected, subprotocol = await communicator.connect()\n self.assertTrue(connected)\n\n asset = await sync_to_async(create_asset)()\n response = await communicator.receive_json_from()\n message = response[\"message\"]\n self.assertEqual(message[\"type\"], \"asset_update\")\n self.assertEqual(message[\"asset_pk\"], asset.pk)\n self.assertEqual(message[\"status\"], \"not_started\")\n self.assertEqual(message[\"latest_transcription\"], None)\n\n await sync_to_async(create_item)(item_id=\"item-2\", project=asset.item.project)\n response = await communicator.receive_nothing()\n self.assertTrue(response)\n\n user = await sync_to_async(self.create_test_user)()\n transcription = await sync_to_async(create_transcription)(\n asset=asset, user=user\n )\n response = await communicator.receive_json_from()\n message = response[\"message\"]\n self.assertEqual(message[\"type\"], \"asset_update\")\n self.assertEqual(message[\"asset_pk\"], asset.pk)\n self.assertEqual(message[\"status\"], \"in_progress\")\n self.assertEqual(message[\"latest_transcription\"][\"id\"], transcription.pk)\n\n await communicator.disconnect()\n\n async def test_asset_reservation_obtained(self):\n asset = await sync_to_async(create_asset)()\n communicator = WebsocketCommunicator(\n AssetConsumer.as_asgi(), \"ws/asset/asset_updates/\"\n )\n connected, subprotocol = await communicator.connect()\n self.assertTrue(connected)\n\n request_factory = RequestFactory()\n request = request_factory.get(\"/\")\n request.session = {}\n token = get_or_create_reservation_token(request)\n await sync_to_async(obtain_reservation)(asset.pk, token)\n\n response = await communicator.receive_json_from()\n message = response[\"message\"]\n self.assertEqual(message[\"type\"], \"asset_reservation_obtained\")\n self.assertEqual(message[\"asset_pk\"], asset.pk)\n\n await communicator.disconnect()\n\n async def test_asset_reservation_released(self):\n asset = await sync_to_async(create_asset)()\n await self.async_client.get(\n reverse(\"reserve-asset\", kwargs={\"asset_pk\": asset.pk})\n )\n\n communicator = WebsocketCommunicator(\n AssetConsumer.as_asgi(), \"ws/asset/asset_updates/\"\n )\n connected, subprotocol = await communicator.connect()\n self.assertTrue(connected)\n\n await self.async_client.post(\n reverse(\"reserve-asset\", kwargs={\"asset_pk\": asset.pk}),\n {\"release\": \"release\"},\n )\n response = await communicator.receive_json_from()\n message = response[\"message\"]\n self.assertEqual(message[\"type\"], \"asset_reservation_released\")\n self.assertEqual(message[\"asset_pk\"], asset.pk)\n await communicator.disconnect()\n"
},
{
"path": "concordia/tests/test_contextmanagers.py",
"content": "from unittest import TestCase\nfrom unittest.mock import patch\n\nfrom concordia.contextmanagers import DEFAULT_LOCK_DURATION, cache_lock\n\n\nclass CacheLockTests(TestCase):\n def setUp(self):\n self.lock_id = \"test-lock\"\n self.oid = \"worker-1\"\n\n self.cache_patch = patch(\"concordia.contextmanagers.cache\")\n self.mock_cache = self.cache_patch.start()\n self.addCleanup(self.cache_patch.stop)\n\n self.time_patch = patch(\"concordia.contextmanagers.time.monotonic\")\n self.mock_monotonic = self.time_patch.start()\n self.addCleanup(self.time_patch.stop)\n\n self.start_time = 100.0\n self.mock_monotonic.return_value = self.start_time\n\n def test_acquires_and_releases_lock(self):\n self.mock_cache.add.return_value = True\n\n with cache_lock(self.lock_id, self.oid) as acquired:\n self.assertTrue(acquired)\n self.mock_cache.add.assert_called_once_with(\n self.lock_id, self.oid, DEFAULT_LOCK_DURATION\n )\n\n self.mock_cache.delete.assert_called_once_with(self.lock_id)\n\n def test_does_not_release_if_lock_not_acquired(self):\n self.mock_cache.add.return_value = False\n\n with cache_lock(self.lock_id, self.oid) as acquired:\n self.assertFalse(acquired)\n\n self.mock_cache.delete.assert_not_called()\n\n def test_does_not_release_if_expired(self):\n self.mock_cache.add.return_value = True\n\n # Simulate expiration: time has passed beyond timeout\n def advance_time():\n return self.start_time + DEFAULT_LOCK_DURATION + 1\n\n self.mock_monotonic.side_effect = [self.start_time, advance_time()]\n\n with cache_lock(self.lock_id, self.oid) as acquired:\n self.assertTrue(acquired)\n\n self.mock_cache.delete.assert_not_called()\n"
},
{
"path": "concordia/tests/test_decorators.py",
"content": "from unittest import TestCase\nfrom unittest.mock import MagicMock, patch\n\nfrom celery import Task\n\nfrom concordia.decorators import locked_task\n\n\nclass LockedTaskDecoratorTests(TestCase):\n def setUp(self):\n self.hostname = \"test-worker\"\n self.logger_patch = patch(\"concordia.decorators.logger\")\n self.logger = self.logger_patch.start()\n self.addCleanup(self.logger_patch.stop)\n\n self.cache_lock_patch = patch(\"concordia.decorators.cache_lock\")\n self.mock_cache_lock = self.cache_lock_patch.start()\n self.addCleanup(self.cache_lock_patch.stop)\n\n def make_task_instance(self, name=\"test-task\"):\n task = MagicMock(spec=Task)\n task.name = name\n task.request.hostname = self.hostname\n return task\n\n def test_lock_by_args_allows_only_one_execution(self):\n task = self.make_task_instance()\n\n calls = []\n\n @locked_task\n def dummy(self, arg):\n calls.append(arg)\n return f\"Ran with {arg}\"\n\n dummy_task = dummy.__get__(task)\n\n self.mock_cache_lock.return_value.__enter__.return_value = True\n result = dummy_task(\"foo\")\n self.assertEqual(result, \"Ran with foo\")\n self.assertEqual(calls, [\"foo\"])\n\n self.mock_cache_lock.return_value.__enter__.return_value = False\n result = dummy_task(\"foo\")\n self.assertIsNone(result)\n self.logger.info.assert_called_once()\n\n def test_lock_by_task_name(self):\n task = self.make_task_instance()\n\n calls = []\n\n @locked_task(lock_by_args=False)\n def dummy(self, arg):\n calls.append(arg)\n return f\"Ran with {arg}\"\n\n dummy_task = dummy.__get__(task)\n\n self.mock_cache_lock.return_value.__enter__.return_value = True\n result = dummy_task(\"foo\")\n self.assertEqual(result, \"Ran with foo\")\n self.assertEqual(calls, [\"foo\"])\n\n def test_force_runs_even_if_lock_not_acquired(self):\n task = self.make_task_instance()\n\n calls = []\n\n @locked_task\n def dummy(self, arg):\n calls.append(arg)\n return f\"Forced {arg}\"\n\n dummy_task = dummy.__get__(task)\n\n self.mock_cache_lock.return_value.__enter__.return_value = False\n result = dummy_task(\"bar\", force=True)\n self.assertEqual(result, \"Forced bar\")\n self.logger.warning.assert_called_once()\n\n def test_error_in_key_generation_logs_and_raises(self):\n task = self.make_task_instance()\n\n @locked_task\n def dummy(self, arg):\n return \"This shouldn't run\"\n\n dummy_task = dummy.__get__(task)\n\n # Use a non-repr-able object to simulate key generation failure\n class Unreprable:\n def __repr__(self):\n raise ValueError(\"Can't repr\")\n\n with self.assertRaises(ValueError):\n dummy_task(Unreprable())\n\n self.logger.exception.assert_called_once_with(\n \"Unable to create cache key from arguments for %s.\", task.name\n )\n"
},
{
"path": "concordia/tests/test_fields.py",
"content": "from unittest import mock\nfrom urllib.error import HTTPError\n\nfrom django.forms import ValidationError\nfrom django.test import TestCase, override_settings\n\nfrom concordia.turnstile.fields import TurnstileField\n\n\nclass TestFields(TestCase):\n @override_settings(\n TURNSTILE_PROXIES={},\n TURNSTILE_SECRET=\"test-secret\", # nosec B106: test-only dummy secret\n TURNSTILE_VERIFY_URL=\"http://example.com\",\n TURNSTILE_TIMEOUT=0,\n )\n def test_TurnstileField(self):\n with (\n override_settings(\n TURNSTILE_DEFAULT_CONFIG={\"appearance\": \"interaction-only\"}\n ),\n mock.patch(\"concordia.turnstile.fields.Request\"),\n mock.patch(\"concordia.turnstile.fields.build_opener\") as opener_mock,\n ):\n open_mock = opener_mock.return_value.open\n read_mock = open_mock.return_value.read\n\n field = TurnstileField(required=False)\n\n self.assertEqual(\n field.widget_attrs(field.widget),\n {\"data-appearance\": \"interaction-only\"},\n )\n\n # Successful validation from Turnstile\n read_mock.return_value = '{\"success\" : true}'.encode()\n self.assertEqual(field.validate(\"test-value\"), None)\n\n # Unsuccessful validation from Turnstile\n read_mock.return_value = '{\"test-key\" : \"test-value\"}'.encode()\n self.assertRaises(ValidationError, field.validate, \"test-value\")\n\n # Error trying to contact Turnstile\n open_mock.side_effect = HTTPError(\n \"http://example.com\", 404, \"Test message%\", \"\", mock.MagicMock()\n )\n self.assertRaises(ValidationError, field.validate, \"test-value\")\n\n # Testing special parameters on the widget\n field = TurnstileField(\n onload=\"testOnload()\",\n render=\"test-render\",\n hl=\"test-hl\",\n test_parameter=\"test-data\",\n )\n self.assertEqual(\n field.widget_attrs(field.widget),\n {\n \"data-appearance\": \"interaction-only\",\n \"data-test_parameter\": \"test-data\",\n },\n )\n self.assertEqual(\n field.widget.extra_url,\n {\n \"onload\": \"testOnload()\",\n \"render\": \"test-render\",\n \"hl\": \"test-hl\",\n },\n )\n"
},
{
"path": "concordia/tests/test_logging.py",
"content": "import warnings\nfrom types import SimpleNamespace\nfrom unittest.mock import MagicMock, patch\n\nfrom django.test import TestCase\n\nfrom concordia.logging import ConcordiaLogger\n\n\nclass ConcordiaLoggerTests(TestCase):\n def setUp(self):\n self.mock_structlog_logger = MagicMock()\n self.logger = ConcordiaLogger(self.mock_structlog_logger)\n\n def test_debug_logs_with_event(self):\n self.logger.debug(\"debug msg\", event_code=\"debug_event\", key1=\"value1\")\n self.mock_structlog_logger.debug.assert_called_once()\n args, kwargs = self.mock_structlog_logger.debug.call_args\n self.assertEqual(args[0], \"debug msg\")\n self.assertEqual(kwargs[\"event_code\"], \"debug_event\")\n self.assertEqual(kwargs[\"key1\"], \"value1\")\n\n def test_info_logs_with_event(self):\n self.logger.info(\"info msg\", event_code=\"info_event\", key2=\"value2\")\n self.mock_structlog_logger.info.assert_called_once()\n args, kwargs = self.mock_structlog_logger.info.call_args\n self.assertEqual(args[0], \"info msg\")\n self.assertEqual(kwargs[\"event_code\"], \"info_event\")\n self.assertEqual(kwargs[\"key2\"], \"value2\")\n\n def test_warning_requires_reason_and_reason_code(self):\n with self.assertRaises(TypeError):\n self.logger.warning(\n \"warning msg\", event_code=\"warn_event\", reason=\"only_reason\"\n )\n\n self.logger.warning(\n \"warning msg\",\n event_code=\"warn_event\",\n reason=\"test reason\",\n reason_code=\"warn_code\",\n key3=\"value3\",\n )\n self.mock_structlog_logger.warning.assert_called_once()\n args, kwargs = self.mock_structlog_logger.warning.call_args\n self.assertEqual(kwargs[\"event_code\"], \"warn_event\")\n self.assertEqual(kwargs[\"reason\"], \"test reason\")\n self.assertEqual(kwargs[\"reason_code\"], \"warn_code\")\n self.assertEqual(kwargs[\"key3\"], \"value3\")\n\n def test_error_requires_reason_and_reason_code(self):\n with self.assertRaises(TypeError):\n self.logger.error(\n \"error msg\", event_code=\"error_event\", reason_code=\"only_code\"\n )\n\n self.logger.error(\n \"error msg\",\n event_code=\"error_event\",\n reason=\"error reason\",\n reason_code=\"error_code\",\n )\n self.mock_structlog_logger.error.assert_called_once()\n args, kwargs = self.mock_structlog_logger.error.call_args\n self.assertEqual(kwargs[\"event_code\"], \"error_event\")\n self.assertEqual(kwargs[\"reason\"], \"error reason\")\n self.assertEqual(kwargs[\"reason_code\"], \"error_code\")\n\n def test_missing_event_raises(self):\n with self.assertRaises(ValueError):\n self.logger.info(\"msg\", event_code=None)\n\n def test_log_merges_context_correctly(self):\n mock_obj = SimpleNamespace(id=42)\n self.logger.register_extractor(\"thing\", lambda o: {\"thing_id\": o.id})\n self.logger.info(\"msg\", event_code=\"test_event\", thing=mock_obj)\n args, kwargs = self.mock_structlog_logger.info.call_args\n self.assertEqual(kwargs[\"thing_id\"], 42)\n\n def test_log_explicit_key_overrides_extracted(self):\n mock_obj = SimpleNamespace(id=42)\n self.logger.register_extractor(\"thing\", lambda o: {\"thing_id\": 123})\n self.logger.info(\"msg\", event_code=\"test_event\", thing=mock_obj, thing_id=999)\n args, kwargs = self.mock_structlog_logger.info.call_args\n self.assertEqual(kwargs[\"thing_id\"], 999)\n\n def test_bind_merges_context(self):\n bound = self.logger.bind(foo=\"bar\")\n self.assertIsInstance(bound, ConcordiaLogger)\n self.assertIn(\"foo\", bound._context)\n self.assertEqual(bound._context[\"foo\"], \"bar\")\n\n def test_bind_merges_context_into_logging(self):\n bound = self.logger.bind(user=\"uval\")\n bound.register_extractor(\"user\", lambda o: {\"user_id\": o})\n bound.info(\"msg\", event_code=\"bound_event\")\n args, kwargs = self.mock_structlog_logger.info.call_args\n self.assertEqual(kwargs[\"user_id\"], \"uval\")\n\n def test_unregister_extractor_removes_extractor(self):\n self.logger.register_extractor(\"foo\", lambda o: {\"foo_id\": 1})\n self.logger.unregister_extractor(\"foo\")\n self.assertNotIn(\"foo\", self.logger._extractors)\n\n def test_register_extractor_warns_on_chained_override(self):\n def fake_asset_extractor(x):\n return {\"asset_id\": 1, \"item_id\": 2}\n\n with warnings.catch_warnings(record=True) as w:\n warnings.simplefilter(\"always\")\n self.logger.register_extractor(\"asset\", fake_asset_extractor)\n self.assertTrue(\n any(\n \"default extractors may still reference\" in str(warn.message)\n for warn in w\n )\n )\n\n def test_log_raises_when_message_is_none(self):\n with self.assertRaises(ValueError):\n self.logger.info(None, event_code=\"event\")\n\n def test_log_raises_when_message_is_none_direct(self):\n with self.assertRaises(ValueError):\n self.logger.log(\"info\", None, event_code=\"event\")\n\n def test_log_raises_when_message_is_empty(self):\n with self.assertRaises(ValueError):\n self.logger.info(\"\", event_code=\"event\")\n\n def test_log_raises_when_message_is_empty_direct(self):\n with self.assertRaises(ValueError):\n self.logger.log(\"info\", \"\", event_code=\"event\")\n\n def test_log_skips_none_values_from_extractor(self):\n class Dummy:\n def __init__(self):\n self.id = None\n\n self.logger.register_extractor(\"thing\", lambda o: {\"thing_id\": o.id})\n self.logger.info(\"msg\", event_code=\"event\", thing=Dummy())\n args, kwargs = self.mock_structlog_logger.info.call_args\n self.assertNotIn(\"thing_id\", kwargs)\n\n def test_log_includes_nonextractor_bound_context(self):\n bound = self.logger.bind(extra1=\"foo\", extra2=\"bar\")\n bound.info(\"msg\", event_code=\"event\")\n args, kwargs = self.mock_structlog_logger.info.call_args\n self.assertEqual(kwargs[\"extra1\"], \"foo\")\n self.assertEqual(kwargs[\"extra2\"], \"bar\")\n\n def test_log_skips_none_values_in_context(self):\n self.logger.info(\"msg\", event_code=\"event\", explicit=None)\n args, kwargs = self.mock_structlog_logger.info.call_args\n self.assertNotIn(\"explicit\", kwargs)\n\n def test_log_overrides_bound_and_extracted_context(self):\n obj = SimpleNamespace(id=123)\n base = self.logger.bind(thing=obj, value=\"a\")\n base.register_extractor(\"thing\", lambda o: {\"thing_id\": o.id, \"value\": \"b\"})\n base.info(\"msg\", event_code=\"event\", value=\"c\")\n args, kwargs = self.mock_structlog_logger.info.call_args\n # Extracted value (\"b\") overridden by explicit context (\"c\")\n self.assertEqual(kwargs[\"value\"], \"c\")\n self.assertEqual(kwargs[\"thing_id\"], 123)\n\n def test_extractor_returns_none_value_skipped(self):\n obj = SimpleNamespace()\n self.logger.register_extractor(\"thing\", lambda o: {\"thing_id\": None})\n self.logger.info(\"msg\", event_code=\"event\", thing=obj)\n args, kwargs = self.mock_structlog_logger.info.call_args\n self.assertNotIn(\"thing_id\", kwargs)\n\n def test_get_logger_uses_structlog(self):\n with patch(\"concordia.logging.structlog.get_logger\") as mock_get_logger:\n mock_logger_instance = MagicMock()\n mock_get_logger.return_value = mock_logger_instance\n\n logger = ConcordiaLogger.get_logger(\"concordia.tests\")\n\n mock_get_logger.assert_called_once_with(\"structlog.concordia.tests\")\n self.assertIsInstance(logger, ConcordiaLogger)\n self.assertEqual(logger._logger, mock_logger_instance)\n\n def test_log_raises_valueerror_for_empty_reason_and_code(self):\n with self.assertRaises(ValueError):\n self.logger.log(\n \"warning\", \"bad\", event_code=\"something\", reason=\"\", reason_code=\"fail\"\n )\n\n with self.assertRaises(ValueError):\n self.logger.log(\n \"error\", \"bad\", event_code=\"something\", reason=\"fail\", reason_code=None\n )\n\n def test_exception_logs_with_exc_info(self):\n try:\n raise ValueError(\"Something went wrong\")\n except ValueError:\n self.logger.exception(\n \"Exception occurred\",\n event_code=\"test_exception\",\n reason=\"An error was raised\",\n reason_code=\"value_error\",\n extra=\"context\",\n )\n\n self.mock_structlog_logger.error.assert_called_once()\n args, kwargs = self.mock_structlog_logger.error.call_args\n self.assertEqual(args[0], \"Exception occurred\")\n self.assertEqual(kwargs[\"event_code\"], \"test_exception\")\n self.assertEqual(kwargs[\"reason\"], \"An error was raised\")\n self.assertEqual(kwargs[\"reason_code\"], \"value_error\")\n self.assertEqual(kwargs[\"extra\"], \"context\")\n self.assertTrue(kwargs.get(\"exc_info\"))\n"
},
{
"path": "concordia/tests/test_maintenance.py",
"content": "from django.core.cache import cache\nfrom django.test import RequestFactory, TestCase\nfrom maintenance_mode.core import set_maintenance_mode\n\nfrom concordia.maintenance import need_maintenance_response\n\nfrom .utils import CreateTestUsers\n\n\nclass TestMaintenance(TestCase, CreateTestUsers):\n def setUp(self):\n self.request_factory = RequestFactory()\n cache.clear()\n\n def tearDown(self):\n cache.clear()\n\n def test_need_maintenance_response_maintenance_default(self):\n request = self.request_factory.get(\"/\")\n self.assertFalse(need_maintenance_response(request))\n\n def test_need_maintenance_response_maintenance_off(self):\n set_maintenance_mode(False)\n request = self.request_factory.get(\"/\")\n self.assertFalse(need_maintenance_response(request))\n\n def test_need_maintenance_response_maintenance_on(self):\n set_maintenance_mode(True)\n request = self.request_factory.get(\"/\")\n self.assertTrue(need_maintenance_response(request))\n\n request.user = self.create_test_user()\n request.user.is_staff = True\n\n # User is set and is staff, but frontend is off\n # (the default) so they should still get a maintenance\n # mode response\n self.assertTrue(need_maintenance_response(request))\n\n def test_need_maintenance_response_maintenance_frontend(self):\n set_maintenance_mode(True)\n request = self.request_factory.get(\"/\")\n request.user = self.create_test_user()\n cache.set(\"maintenance_mode_frontend_available\", True)\n\n # User is set but isn't super user, so they should get\n # a maintenance mode response\n self.assertTrue(need_maintenance_response(request))\n\n request.user.is_staff = True\n # User is staff, so they shouldn't get a maintenance\n # mode response\n self.assertFalse(need_maintenance_response(request))\n"
},
{
"path": "concordia/tests/test_management_commands.py",
"content": "from io import StringIO\nfrom unittest import mock\n\nfrom django.core.management import call_command\nfrom django.test import TestCase\n\nfrom concordia.tests.utils import create_asset, create_campaign\n\n\nclass EnsureInitialSiteConfigurationTests(TestCase):\n def test_command_output(self, *args, **kwargs):\n out = StringIO()\n call_command(\n \"ensure_initial_site_configuration\", admin_email=\"admin@loc.gov\", stdout=out\n )\n call_command(\n \"ensure_initial_site_configuration\", site_domain=\"crowd.loc.gov\", stdout=out\n )\n with mock.patch(\n \"django.contrib.sites.models.Site.objects.update\"\n ) as update_mock:\n update_mock.return_value = 0\n call_command(\n \"ensure_initial_site_configuration\",\n site_domain=\"crowd.loc.gov\",\n stdout=out,\n )\n\n\nclass ImportSiteReportsTests(TestCase):\n def test_command_output(self, *args, **kwargs):\n out = StringIO()\n create_campaign(id=1)\n call_command(\n \"import_site_reports\",\n csv_file=\"concordia/tests/data/site_reports.csv\",\n stdout=out,\n )\n\n\nclass PrintFrontendTestUrlsTests(TestCase):\n def test_command_output(self, *args, **kwargs):\n out = StringIO()\n call_command(\"print_frontend_test_urls\", stdout=out)\n self.assertIn(\"\", out.getvalue())\n\n create_asset()\n call_command(\"print_frontend_test_urls\", stdout=out)\n self.assertIn(\"\", out.getvalue())\n"
},
{
"path": "concordia/tests/test_models.py",
"content": "import json\nfrom datetime import date, datetime, timedelta\nfrom decimal import Decimal\nfrom secrets import token_hex\nfrom unittest import mock\n\nfrom django.conf import settings\nfrom django.core.exceptions import ObjectDoesNotExist, ValidationError\nfrom django.db.models import signals\nfrom django.test import TestCase\nfrom django.utils import timezone\n\nfrom concordia.models import (\n Asset,\n AssetTranscriptionReservation,\n Campaign,\n CardFamily,\n ConcordiaUser,\n HelpfulLink,\n KeyMetricsReport,\n MediaType,\n NextReviewableCampaignAsset,\n NextReviewableTopicAsset,\n NextTranscribableCampaignAsset,\n NextTranscribableTopicAsset,\n SiteReport,\n Topic,\n Transcription,\n TranscriptionStatus,\n UserProfile,\n UserProfileActivity,\n _update_useractivity_cache,\n resource_file_upload_path,\n update_userprofileactivity_table,\n validated_get_or_create,\n)\nfrom concordia.signals.handlers import create_user_profile, on_transcription_save\nfrom concordia.utils import get_anonymous_user\n\nfrom .utils import (\n CreateTestUsers,\n create_asset,\n create_banner,\n create_campaign,\n create_campaign_retirement_progress,\n create_card,\n create_card_family,\n create_carousel_slide,\n create_concordia_file,\n create_guide,\n create_helpful_link,\n create_item,\n create_simple_page,\n create_tag,\n create_tag_collection,\n create_topic,\n create_transcription,\n create_user_profile_activity,\n)\n\n\nclass AssetTestCase(CreateTestUsers, TestCase):\n def setUp(self):\n self.asset = create_asset()\n self.anon = get_anonymous_user()\n create_transcription(asset=self.asset, user=self.anon)\n create_transcription(\n asset=self.asset,\n user=self.create_test_user(username=\"tester\"),\n reviewed_by=self.anon,\n )\n\n def test_get_ocr_transcript(self):\n self.asset.storage_image = \"tests/test-european.jpg\"\n self.asset.save()\n phrase = \"marrΓ³n rΓ‘pido salta sobre el perro\"\n self.assertFalse(phrase in self.asset.get_ocr_transcript())\n self.assertFalse(\n phrase in self.asset.get_ocr_transcript(language=\"bad-language-code\")\n )\n self.assertTrue(phrase in self.asset.get_ocr_transcript(language=\"spa\"))\n\n def test_get_contributor_count(self):\n self.assertEqual(self.asset.get_contributor_count(), 2)\n\n def test_turn_off_ocr(self):\n self.assertFalse(self.asset.turn_off_ocr())\n self.asset.disable_ocr = True\n self.asset.save()\n self.assertTrue(self.asset.turn_off_ocr())\n\n self.assertFalse(self.asset.item.turn_off_ocr())\n self.asset.item.disable_ocr = True\n self.asset.item.save()\n self.assertTrue(self.asset.item.turn_off_ocr())\n\n self.assertFalse(self.asset.item.project.turn_off_ocr())\n self.asset.item.project.disable_ocr = True\n self.asset.item.project.save()\n self.assertTrue(self.asset.item.project.turn_off_ocr())\n\n def test_get_storage_path(self):\n self.assertEqual(\n self.asset.get_storage_path(filename=self.asset.storage_image.name),\n \"test-campaign/test-project/testitem.0123456789/1.jpg\",\n )\n\n def test_saving_without_campaign(self):\n try:\n Asset.objects.create(\n item=self.asset.item,\n title=\"No campaign\",\n slug=\"no-campaign\",\n media_type=MediaType.IMAGE,\n storage_image=\"unittest1.jpg\",\n )\n except (ValidationError, ObjectDoesNotExist):\n self.fail(\"Creating an Asset without a campaign failed validation.\")\n\n def test_rollforward_with_only_rollforward_transcriptions(self):\n asset = create_asset(slug=\"rollforward-test\", item=self.asset.item)\n create_transcription(asset=asset, user=self.anon, rolled_forward=True)\n with self.assertRaisesMessage(\n ValueError,\n \"Can not rollforward transcription on an asset with \"\n \"no non-rollforward transcriptions\",\n ):\n asset.rollforward_transcription(self.anon)\n\n def test_rollforward_with_too_many_rollforward_transcriptions(self):\n asset = create_asset(slug=\"rollforward-test\", item=self.asset.item)\n transcription1 = create_transcription(asset=asset, user=self.anon)\n create_transcription(\n asset=asset, user=self.anon, supersedes=transcription1, rolled_forward=True\n )\n create_transcription(\n asset=asset, user=self.anon, supersedes=transcription1, rolled_forward=True\n )\n with self.assertRaisesMessage(\n ValueError,\n \"More rollforward transcription exist than non-roll-forward \"\n \"transcriptions, which shouldn't be possible. Possibly \"\n \"incorrectly modified transcriptions for this asset.\",\n ):\n asset.rollforward_transcription(self.anon)\n\n def test_rollforward_with_no_superseded_transcription(self):\n # This isn't a state that would happen normally, but could be created\n # accidentally when manually editing transcription history\n asset = create_asset(slug=\"rollforward-test\", item=self.asset.item)\n transcription1 = create_transcription(asset=asset, user=self.anon)\n create_transcription(asset=asset, user=self.anon, supersedes=transcription1)\n create_transcription(\n asset=asset, user=self.anon, rolled_back=True, source=transcription1\n )\n with self.assertRaisesMessage(\n ValueError,\n \"Can not rollforward transcription on an asset if the latest \"\n \"rollback transcription did not supersede a previous transcription\",\n ):\n asset.rollforward_transcription(self.anon)\n\n def test_get_storage_path_handles_jpeg(self):\n # Ensure \".jpeg\" is normalized to \".jpg\"\n expected = self.asset.get_asset_image_filename(\"jpg\")\n self.assertEqual(self.asset.get_storage_path(\"anything.jpeg\"), expected)\n\n\nclass ItemModelTests(TestCase):\n def test_thumbnail_link_prefers_image_url_when_present(self):\n item = create_item()\n\n class Img:\n url = \"http://example.test/media/thumb.jpg\"\n\n item.thumbnail_image = Img()\n self.assertEqual(item.thumbnail_link, Img.url)\n\n def test_thumbnail_link_falls_back_when_image_url_raises(self):\n # If .url access raises ValueError, fall back to thumbnail_url\n item = create_item()\n\n class BadImg:\n @property\n def url(self):\n raise ValueError(\"missing from storage\")\n\n item.thumbnail_image = BadImg()\n item.thumbnail_url = \"http://example.test/media/fallback.jpg\"\n self.assertEqual(item.thumbnail_link, item.thumbnail_url)\n\n def test_thumbnail_link_returns_thumbnail_url_when_no_image(self):\n item = create_item()\n item.thumbnail_image = None\n item.thumbnail_url = \"http://example.test/media/fallback.jpg\"\n self.assertEqual(item.thumbnail_link, item.thumbnail_url)\n\n def test_thumbnail_link_returns_none_when_no_image_or_url(self):\n item = create_item()\n item.thumbnail_image = None\n item.thumbnail_url = None\n self.assertIsNone(item.thumbnail_link)\n\n\nclass TranscriptionManagerTestCase(CreateTestUsers, TestCase):\n def setUp(self):\n self.transcription1 = create_transcription(\n user=self.create_user(username=\"tester1\"),\n rejected=timezone.now() - timedelta(days=2),\n )\n self.transcription2 = create_transcription(\n asset=self.transcription1.asset, user=get_anonymous_user()\n )\n\n def test_recent_review_actions(self):\n transcriptions = Transcription.objects\n self.assertEqual(transcriptions.recent_review_actions().count(), 0)\n\n self.transcription1.accepted = timezone.now()\n self.transcription1.save()\n self.assertEqual(transcriptions.recent_review_actions().count(), 1)\n\n self.transcription2.rejected = timezone.now()\n self.transcription2.save()\n self.assertEqual(transcriptions.recent_review_actions().count(), 2)\n\n def test_review_actions(self):\n start = timezone.now() - timedelta(days=5)\n end = timezone.now() - timedelta(days=1)\n self.assertEqual(Transcription.objects.review_actions(start, end).count(), 1)\n\n def test_review_incidents(self):\n self.transcription1.accepted = timezone.now()\n self.transcription1.reviewed_by = self.create_user(username=\"tester2\")\n self.transcription1.save()\n self.transcription2.accepted = self.transcription1.accepted + timedelta(\n seconds=29\n )\n self.transcription2.reviewed_by = self.transcription1.reviewed_by\n self.transcription2.save()\n create_transcription(\n asset=self.transcription1.asset,\n user=self.transcription1.user,\n reviewed_by=self.transcription1.reviewed_by,\n rejected=self.transcription2.accepted + timedelta(seconds=29),\n )\n create_transcription(\n asset=self.transcription1.asset,\n user=self.transcription1.user,\n reviewed_by=self.transcription1.reviewed_by,\n rejected=self.transcription2.accepted + timedelta(seconds=58),\n )\n users = Transcription.objects.review_incidents()\n self.assertNotIn(self.transcription1.user.id, users)\n\n transcription3 = create_transcription(\n asset=self.transcription1.asset,\n user=self.transcription1.user,\n reviewed_by=self.transcription1.reviewed_by,\n accepted=self.transcription1.accepted + timedelta(seconds=58),\n )\n transcription4 = create_transcription(\n asset=self.transcription1.asset,\n user=self.transcription1.user,\n reviewed_by=self.transcription1.reviewed_by,\n accepted=transcription3.accepted + timedelta(minutes=1, seconds=1),\n )\n users = Transcription.objects.review_incidents()\n self.assertEqual(len(users), 1)\n self.assertEqual(\n users[0],\n (\n self.transcription1.reviewed_by.id,\n self.transcription1.reviewed_by.username,\n 2,\n 4,\n ),\n )\n\n create_transcription(\n asset=self.transcription1.asset,\n user=self.transcription1.user,\n reviewed_by=self.transcription1.reviewed_by,\n accepted=transcription4.accepted + timedelta(seconds=29),\n )\n create_transcription(\n asset=self.transcription1.asset,\n user=self.transcription1.user,\n reviewed_by=self.transcription1.reviewed_by,\n accepted=transcription4.accepted + timedelta(seconds=58),\n )\n users = Transcription.objects.review_incidents()\n self.assertEqual(len(users), 1)\n self.assertEqual(\n users[0],\n (\n self.transcription1.reviewed_by.id,\n self.transcription1.reviewed_by.username,\n 4,\n 6,\n ),\n )\n\n def test_transcribe_incidents(self):\n self.transcription1.submitted = timezone.now()\n self.transcription1.save()\n self.transcription2.submitted = self.transcription1.submitted + timedelta(\n seconds=29\n )\n self.transcription2.user = self.transcription1.user\n self.transcription2.save()\n users = Transcription.objects.transcribe_incidents()\n self.assertEqual(len(users), 0)\n self.assertNotIn(self.transcription1.user.id, users)\n\n transcription3 = create_transcription(\n asset=create_asset(slug=\"asset-two\", item=self.transcription1.asset.item),\n user=self.transcription1.user,\n submitted=self.transcription1.submitted + timedelta(seconds=58),\n )\n transcription4 = create_transcription(\n asset=create_asset(slug=\"asset-three\", item=self.transcription1.asset.item),\n user=self.transcription1.user,\n submitted=transcription3.submitted + timedelta(minutes=1, seconds=1),\n )\n create_transcription(\n asset=transcription4.asset,\n user=self.transcription1.user,\n submitted=transcription4.submitted + timedelta(seconds=59),\n )\n users = Transcription.objects.transcribe_incidents()\n self.assertEqual(len(users), 1)\n self.assertEqual(\n users[0],\n (self.transcription1.user.id, self.transcription1.user.username, 2, 5),\n )\n\n create_transcription(\n asset=create_asset(slug=\"asset-five\", item=self.transcription1.asset.item),\n user=self.transcription1.user,\n submitted=self.transcription1.submitted + timedelta(minutes=1, seconds=59),\n )\n users = Transcription.objects.transcribe_incidents()\n self.assertEqual(len(users), 1)\n self.assertEqual(\n users[0],\n (self.transcription1.user.id, self.transcription1.user.username, 3, 6),\n )\n\n def test_review_incidents_returns_empty_when_counts_zero(self):\n reviewer = self.create_user(username=\"rev-zero\")\n asset = self.transcription1.asset\n\n t1 = create_transcription(\n asset=asset,\n user=self.create_user(username=\"u-a\"),\n reviewed_by=reviewer,\n accepted=timezone.now() - timedelta(minutes=5),\n )\n create_transcription(\n asset=asset,\n user=self.create_user(username=\"u-b\"),\n reviewed_by=reviewer,\n accepted=t1.accepted + timedelta(seconds=61),\n )\n\n out = Transcription.objects.review_incidents()\n self.assertEqual(out, [])\n\n def test_user_review_incidents_no_threshold_hit(self):\n asset = self.transcription1.asset\n\n reviewer = self.create_user(\"reviewer-1\")\n reviewer_proxy = ConcordiaUser.objects.get(pk=reviewer.pk)\n\n base = timezone.now()\n create_transcription(\n asset=asset,\n user=self.create_user(\"ri_u1\"),\n reviewed_by=reviewer_proxy,\n accepted=base,\n )\n create_transcription(\n asset=asset,\n user=self.create_user(\"ri_u2\"),\n reviewed_by=reviewer_proxy,\n accepted=base + timedelta(seconds=61),\n )\n\n recent = Transcription.objects.filter(accepted__isnull=False)\n incidents = reviewer_proxy.review_incidents(recent)\n self.assertEqual(incidents, 0)\n\n def test_review_incidents_no_threshold_match_inner_loop_break(self):\n # Two accepts for same reviewer but >60s apart:\n a1 = create_asset(slug=\"rev-gap-a1\", item=self.transcription1.asset.item)\n a2 = create_asset(slug=\"rev-gap-a2\", item=a1.item)\n reviewer = self.create_user(\"reviewer-1\")\n\n t0 = timezone.now()\n create_transcription(\n asset=a1, user=self.create_user(\"u1\"), reviewed_by=reviewer, accepted=t0\n )\n create_transcription(\n asset=a2,\n user=self.create_user(\"u2\"),\n reviewed_by=reviewer,\n accepted=t0 + timedelta(seconds=61),\n )\n\n recent = Transcription.objects.filter(accepted__isnull=False)\n reviewer_proxy = ConcordiaUser.objects.get(pk=reviewer.pk)\n\n incidents = reviewer_proxy.review_incidents(recent)\n self.assertEqual(incidents, 0)\n\n def test_review_incidents_loops_until_threshold(self):\n reviewer = self.create_user(username=\"test-reviewer-1\")\n\n # Three accepts within 60s so threshold=3 will require two inner\n # iterations (count goes 1->2, not equal to threshold, then 2->3)\n base = timezone.now()\n create_transcription(\n asset=self.transcription1.asset,\n user=self.transcription1.user,\n reviewed_by=reviewer,\n accepted=base,\n )\n create_transcription(\n asset=self.transcription1.asset,\n user=self.transcription1.user,\n reviewed_by=reviewer,\n accepted=base + timedelta(seconds=20),\n )\n create_transcription(\n asset=self.transcription1.asset,\n user=self.transcription1.user,\n reviewed_by=reviewer,\n accepted=base + timedelta(seconds=40),\n )\n\n recent_accepts = Transcription.objects.filter(\n accepted__gte=base - timedelta(seconds=1)\n )\n\n reviewer_proxy = ConcordiaUser.objects.get(id=reviewer.id)\n\n incidents = reviewer_proxy.review_incidents(\n recent_accepts=recent_accepts, threshold=3\n )\n self.assertEqual(incidents, 1)\n\n\nclass TranscriptionTestCase(CreateTestUsers, TestCase):\n def setUp(self):\n self.user = self.create_user(\"test-user-1\")\n self.user2 = self.create_user(\"test-user-2\")\n self.asset = create_asset()\n self.transcription1 = create_transcription(\n user=self.user,\n asset=self.asset,\n rejected=timezone.now() - timedelta(days=2),\n )\n self.transcription2 = create_transcription(asset=self.asset, user=self.user2)\n\n def test_campaign_slug(self):\n self.assertEqual(\n self.asset.item.project.campaign.slug, self.transcription1.campaign_slug()\n )\n\n def test_clean(self):\n bad_transcription = Transcription(asset=self.asset, user=self.user)\n bad_transcription.clean()\n\n bad_transcription2 = Transcription(\n asset=self.asset,\n user=self.user,\n reviewed_by=self.user,\n accepted=timezone.now(),\n )\n with self.assertRaises(ValidationError):\n bad_transcription2.clean()\n\n bad_transcription3 = Transcription(\n asset=self.asset,\n user=self.user,\n reviewed_by=self.user2,\n accepted=timezone.now(),\n rejected=timezone.now(),\n )\n with self.assertRaises(ValidationError):\n bad_transcription3.clean()\n\n @mock.patch(\"concordia.tests.test_models.on_transcription_save\")\n def test_save(self, mock_handler):\n signals.post_save.connect(on_transcription_save, sender=Transcription)\n\n transcription = create_transcription(asset=self.asset)\n self.assertTrue(mock_handler.called)\n self.assertEqual(mock_handler.call_count, 1)\n\n transcription.save()\n self.assertEqual(mock_handler.call_count, 2)\n\n signals.post_save.disconnect(on_transcription_save, sender=Transcription)\n\n def test_status(self):\n transcription = create_transcription(user=self.user, asset=self.asset)\n self.assertEqual(\n transcription.status,\n TranscriptionStatus.CHOICE_MAP[TranscriptionStatus.IN_PROGRESS],\n )\n\n transcription2 = create_transcription(\n asset=transcription.asset, user=self.user, submitted=timezone.now()\n )\n self.assertEqual(\n transcription2.status,\n TranscriptionStatus.CHOICE_MAP[TranscriptionStatus.SUBMITTED],\n )\n\n transcription3 = create_transcription(\n asset=transcription.asset,\n user=self.user,\n reviewed_by=self.user2,\n accepted=timezone.now(),\n )\n self.assertEqual(\n transcription3.status,\n TranscriptionStatus.CHOICE_MAP[TranscriptionStatus.COMPLETED],\n )\n\n\nclass SignalHandlersTest(CreateTestUsers, TestCase):\n @mock.patch(\"django.core.cache.cache.get\")\n @mock.patch(\"django.core.cache.cache.set\")\n def test_update_useractivity_cache(self, mock_set, mock_get):\n campaign = create_campaign()\n user = self.create_test_user()\n mock_get.return_value = {}\n _update_useractivity_cache(user.id, campaign.id, \"transcribe\")\n self.assertEqual(mock_set.call_count, 1)\n expected_key = f\"userprofileactivity_{campaign.pk}\"\n expected_value = {user.id: (1, 0)}\n mock_set.assert_called_with(expected_key, expected_value, timeout=None)\n\n reviewed_by = self.create_test_user(username=\"testuser2\")\n mock_get.return_value = {}\n _update_useractivity_cache(reviewed_by.id, campaign.id, \"review\")\n self.assertEqual(mock_set.call_count, 2)\n expected_value = {reviewed_by.id: (0, 1)}\n mock_set.assert_called_with(expected_key, expected_value, timeout=None)\n\n\nclass AssetTranscriptionReservationTest(CreateTestUsers, TestCase):\n def setUp(self):\n self.asset = create_asset()\n self.user = self.create_user(\"test-user\")\n self.uid = str(self.user.id).zfill(6)\n self.token = token_hex(22)\n self.reservation_token = self.token + self.uid\n self.reservation = AssetTranscriptionReservation.objects.create(\n asset=self.asset, reservation_token=self.reservation_token\n )\n\n def test_get_token(self):\n self.assertEqual(self.reservation.get_token(), self.token)\n\n def test_get_user(self):\n self.assertEqual(self.reservation.get_user(), self.uid)\n\n\nclass UserProfileActivityTestCase(TestCase):\n def setUp(self):\n self.user_profile_activity = UserProfileActivity(\n campaign=Campaign(), transcribe_count=135, review_count=204\n )\n\n def test_get_status(self):\n self.user_profile_activity.campaign.status = Campaign.Status.ACTIVE\n self.assertEqual(self.user_profile_activity.get_status(), \"Active\")\n self.user_profile_activity.campaign.status = Campaign.Status.COMPLETED\n self.assertEqual(self.user_profile_activity.get_status(), \"Completed\")\n self.user_profile_activity.campaign.status = Campaign.Status.RETIRED\n self.assertEqual(self.user_profile_activity.get_status(), \"Retired\")\n\n def test_total_actions(self):\n self.assertEqual(self.user_profile_activity.total_actions(), 339)\n\n def test_str(self):\n activity = create_user_profile_activity()\n self.assertEqual(f\"{activity.user} - {activity.campaign}\", str(activity))\n\n\nclass UserProfileTestCase(CreateTestUsers, TestCase):\n def test_update_userprofileactivity_table(self):\n signals.post_save.disconnect(\n create_user_profile, sender=settings.AUTH_USER_MODEL\n )\n\n user = self.create_test_user()\n self.assertFalse(hasattr(user, \"profile\"))\n\n transcription = create_transcription(user=user)\n update_userprofileactivity_table(\n user, transcription.asset.item.project.campaign.id, \"transcribe_count\"\n )\n\n self.assertTrue(hasattr(user, \"profile\"))\n self.assertEqual(user.profile.transcribe_count, 1)\n\n signals.post_save.connect(create_user_profile, sender=settings.AUTH_USER_MODEL)\n\n def test_update_userprofileactivity_table_updates_existing_and_profile(self):\n # Avoid auto-profile creation so we control both branches\n signals.post_save.disconnect(\n create_user_profile, sender=settings.AUTH_USER_MODEL\n )\n\n user = self.create_test_user()\n UserProfile.objects.create(user=user)\n\n transcription = create_transcription(user=user)\n campaign = transcription.asset.item.project.campaign\n upa, _ = UserProfileActivity.objects.get_or_create(\n user=user, campaign=campaign, defaults={\"transcribe_count\": 1}\n )\n\n update_userprofileactivity_table(user, campaign.id, \"transcribe_count\")\n\n # F() increments apply on save; refresh to observe DB values\n upa.refresh_from_db()\n user.refresh_from_db()\n user.profile.refresh_from_db()\n\n self.assertEqual(upa.transcribe_count, 2)\n self.assertEqual(user.profile.transcribe_count, 1)\n\n signals.post_save.connect(create_user_profile, sender=settings.AUTH_USER_MODEL)\n\n\nclass CampaignTestCase(TestCase):\n def test_queryset(self):\n campaign = create_campaign(unlisted=True)\n self.assertIn(campaign, Campaign.objects.unlisted())\n\n campaign.status = Campaign.Status.COMPLETED\n campaign.save()\n self.assertIn(campaign, Campaign.objects.completed())\n\n campaign.status = Campaign.Status.RETIRED\n campaign.save()\n self.assertIn(campaign, Campaign.objects.retired())\n\n\nclass CardTestCase(TestCase):\n def test_str(self):\n card = create_card()\n self.assertEqual(card.title, str(card))\n\n\nclass CardFamilyTestCase(TestCase):\n def setUp(self):\n self.family1 = create_card_family(default=True)\n\n def test_str(self):\n self.assertEqual(self.family1.slug, str(self.family1))\n\n def test_on_cardfamily_save(self):\n with mock.patch(\"concordia.models.on_cardfamily_save\") as mocked_handler:\n signals.post_save.connect(mocked_handler, sender=CardFamily)\n self.family1.save()\n self.assertTrue(mocked_handler.called)\n self.assertEqual(mocked_handler.call_count, 1)\n\n\nclass HelpfulLinkTestCase(TestCase):\n def setUp(self):\n self.helpful_link = create_helpful_link()\n\n def test_str(self):\n self.assertEqual(self.helpful_link.title, str(self.helpful_link))\n\n def test_queryset(self):\n self.assertEqual(HelpfulLink.objects.related_links().count(), 1)\n create_helpful_link(\n link_type=HelpfulLink.HelpfulLinkType.COMPLETED_TRANSCRIPTION_LINK\n )\n self.assertEqual(HelpfulLink.objects.completed_transcription_links().count(), 1)\n\n\nclass ConcordiaFileTestCase(TestCase):\n def setUp(self):\n self.concordia_file = create_concordia_file()\n\n def test_str(self):\n self.assertEqual(self.concordia_file.name, str(self.concordia_file))\n\n def test_delete(self):\n with (\n mock.patch.object(\n self.concordia_file.uploaded_file, \"delete\"\n ) as delete_mock,\n mock.patch.object(\n self.concordia_file.uploaded_file, \"storage\", autospec=True\n ) as storage_mock,\n ):\n storage_mock.exists.return_value = True\n self.concordia_file.delete()\n self.assertTrue(delete_mock.called)\n\n concordia_file2 = create_concordia_file()\n with (\n mock.patch.object(concordia_file2.uploaded_file, \"delete\") as delete_mock,\n mock.patch.object(\n concordia_file2.uploaded_file, \"storage\", autospec=True\n ) as storage_mock,\n ):\n storage_mock.exists.return_value = False\n concordia_file2.delete()\n self.assertFalse(delete_mock.called)\n\n def test_concordia_file_upload_path(self):\n current_year = date.today().year\n\n path = resource_file_upload_path(self.concordia_file, \"SHOULDNTBEUSED.PDF\")\n self.assertEqual(path, \"file.pdf\")\n\n self.concordia_file.path = None\n\n path = resource_file_upload_path(self.concordia_file, \"TEST.PDF\")\n self.assertEqual(path, f\"cm-uploads/resources/{current_year}/test.pdf\")\n\n path = resource_file_upload_path(self.concordia_file, \"TEST%%s.PDF\")\n self.assertEqual(path, f\"cm-uploads/resources/{current_year}/test%s.pdf\")\n\n path = resource_file_upload_path(self.concordia_file, \"%%YTEST.PDF\")\n self.assertEqual(path, f\"cm-uploads/resources/{current_year}/%ytest.pdf\")\n\n\nclass TagTestCase(TestCase):\n def test_str(self):\n tag = create_tag()\n self.assertEqual(tag.value, str(tag))\n\n\nclass UserAssetTagCollectionTestCase(TestCase):\n def test_str(self):\n tag_collection = create_tag_collection()\n self.assertEqual(\n \"{} - {}\".format(tag_collection.asset, tag_collection.user),\n str(tag_collection),\n )\n\n\nclass BannerTestCase(TestCase):\n def setUp(self):\n self.banner = create_banner()\n\n def test_str(self):\n self.assertEqual(f\"Banner: {self.banner.slug}\", str(self.banner))\n\n def test_alert_class(self):\n self.assertEqual(\n self.banner.alert_class(), \"alert-\" + self.banner.alert_status.lower()\n )\n\n def test_btn_class(self):\n self.assertEqual(\n self.banner.btn_class(), \"btn-\" + self.banner.alert_status.lower()\n )\n\n\nclass CarouselSlideTestCase(TestCase):\n def test_str(self):\n slide = create_carousel_slide()\n self.assertEqual(f\"CarouselSlide: {slide.headline}\", str(slide))\n\n\nclass CampaignRetirementProgressTestCase(TestCase):\n def test_str(self):\n progress = create_campaign_retirement_progress()\n self.assertEqual(f\"Removal progress for {progress.campaign}\", str(progress))\n\n\nclass GuideTestCase(TestCase):\n def test_str(self):\n guide = create_guide()\n self.assertEqual(guide.title, str(guide))\n\n\nclass SimplePageTestCase(TestCase):\n def test_str(self):\n simple_page = create_simple_page()\n self.assertEqual(f\"SimplePage: {simple_page.path}\", str(simple_page))\n\n\nclass ValidatedGetOrCreateTestCase(TestCase):\n def test_validated_get_or_create(self):\n kwargs = {\n \"title\": \"Test Campaign\",\n \"slug\": \"test-campaign\",\n }\n campaign, created = validated_get_or_create(Campaign, **kwargs)\n self.assertTrue(created)\n campaign, created = validated_get_or_create(Campaign, **kwargs)\n self.assertFalse(created)\n self.assertEqual(campaign.title, kwargs[\"title\"])\n\n\nclass NextAssetModelTests(TestCase):\n def setUp(self):\n self.asset = create_asset()\n self.topic = create_topic(project=self.asset.item.project)\n self.campaign = self.asset.campaign\n self.project = self.asset.item.project\n\n def test_create_next_transcribable_campaign_asset(self):\n obj = NextTranscribableCampaignAsset.objects.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.project,\n project_slug=self.project.slug,\n sequence=self.asset.sequence,\n campaign=self.campaign,\n )\n self.assertEqual(str(obj), self.asset.title)\n self.assertEqual(obj.transcription_status, \"not_started\")\n\n def test_create_next_reviewable_campaign_asset(self):\n obj = NextReviewableCampaignAsset.objects.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.project,\n project_slug=self.project.slug,\n sequence=self.asset.sequence,\n campaign=self.campaign,\n )\n self.assertEqual(str(obj), self.asset.title)\n self.assertEqual(obj.transcriber_ids, [])\n\n def test_create_next_transcribable_topic_asset(self):\n obj = NextTranscribableTopicAsset.objects.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.project,\n project_slug=self.project.slug,\n sequence=self.asset.sequence,\n topic=self.topic,\n )\n self.assertEqual(obj.transcription_status, \"not_started\")\n\n def test_create_next_reviewable_topic_asset(self):\n obj = NextReviewableTopicAsset.objects.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.project,\n project_slug=self.project.slug,\n sequence=self.asset.sequence,\n topic=self.topic,\n )\n self.assertEqual(obj.transcriber_ids, [])\n\n def test_needed_for_campaign_respects_target_count(self):\n manager = NextTranscribableCampaignAsset.objects\n current_needed = manager.needed_for_campaign(self.campaign.id)\n self.assertEqual(current_needed, settings.NEXT_TRANSCRIBABE_ASSET_COUNT)\n\n # Add one and check count again\n manager.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.project,\n project_slug=self.project.slug,\n sequence=self.asset.sequence,\n campaign=self.campaign,\n )\n new_needed = manager.needed_for_campaign(self.campaign.id)\n self.assertEqual(new_needed, settings.NEXT_TRANSCRIBABE_ASSET_COUNT - 1)\n\n def test_needed_for_topic_respects_target_count(self):\n manager = NextReviewableTopicAsset.objects\n current_needed = manager.needed_for_topic(self.topic.id)\n self.assertEqual(current_needed, settings.NEXT_REVIEWABLE_ASSET_COUNT)\n\n manager.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.project,\n project_slug=self.project.slug,\n sequence=self.asset.sequence,\n topic=self.topic,\n )\n new_needed = manager.needed_for_topic(self.topic.id)\n self.assertEqual(new_needed, settings.NEXT_REVIEWABLE_ASSET_COUNT - 1)\n\n def test_needed_for_campaign_raises_without_target(self):\n from django.db import models\n\n from concordia.models import NextCampaignAssetManager\n\n class DummyManager(NextCampaignAssetManager):\n target_count = None\n\n class DummyModel(models.Model):\n campaign = models.ForeignKey(\"concordia.Campaign\", on_delete=models.CASCADE)\n objects = DummyManager()\n\n class Meta:\n app_label = \"concordia\"\n\n with self.assertRaises(NotImplementedError):\n DummyModel.objects.needed_for_campaign(self.campaign.id)\n\n def test_needed_for_topic_raises_without_target(self):\n from django.db import models\n\n from concordia.models import NextTopicAssetManager\n\n class DummyManager(NextTopicAssetManager):\n target_count = None\n\n class DummyModel(models.Model):\n topic = models.ForeignKey(\"concordia.Topic\", on_delete=models.CASCADE)\n objects = DummyManager()\n\n class Meta:\n app_label = \"concordia\"\n\n with self.assertRaises(NotImplementedError):\n DummyModel.objects.needed_for_topic(self.topic.id)\n\n def test_needed_for_campaign_with_explicit_target_count(self):\n manager = NextTranscribableCampaignAsset.objects\n # Should return full count when no assets exist yet\n needed = manager.needed_for_campaign(self.campaign.id, target_count=10)\n self.assertEqual(needed, 10)\n\n # Add one asset\n manager.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.project,\n project_slug=self.project.slug,\n sequence=self.asset.sequence,\n campaign=self.campaign,\n )\n\n needed = manager.needed_for_campaign(self.campaign.id, target_count=10)\n self.assertEqual(needed, 9)\n\n def test_needed_for_topic_with_explicit_target_count(self):\n manager = NextReviewableTopicAsset.objects\n needed = manager.needed_for_topic(self.topic.id, target_count=5)\n self.assertEqual(needed, 5)\n\n manager.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.project,\n project_slug=self.project.slug,\n sequence=self.asset.sequence,\n topic=self.topic,\n )\n\n needed = manager.needed_for_topic(self.topic.id, target_count=5)\n self.assertEqual(needed, 4)\n\n\nclass SiteReportAndManagerTestCase(TestCase):\n def _aware(self, y, m, d, hh=12, mm=0, ss=0):\n tz = timezone.get_current_timezone()\n return timezone.make_aware(datetime(y, m, d, hh, mm, ss), tz)\n\n def _mk_sr(\n self,\n *,\n dt,\n report_name=None,\n campaign=None,\n topic=None,\n **kwargs,\n ):\n sr = SiteReport.objects.create(\n report_name=report_name or \"\",\n campaign=campaign,\n topic=topic,\n **kwargs,\n )\n # Set created_on deterministically for ordering logic\n SiteReport.objects.filter(pk=sr.pk).update(created_on=dt)\n return SiteReport.objects.get(pk=sr.pk)\n\n def test_calculate_assets_started(self):\n # Uses (assets_total - assets_not_started) deltas; floor at 0.\n v = SiteReport.calculate_assets_started(\n previous_assets_total=100,\n previous_assets_not_started=100,\n current_assets_total=107,\n current_assets_not_started=92,\n )\n self.assertEqual(v, 15)\n\n # None treated as 0.\n v2 = SiteReport.calculate_assets_started(\n previous_assets_total=None,\n previous_assets_not_started=None,\n current_assets_total=200,\n current_assets_not_started=190,\n )\n self.assertEqual(v2, 10)\n\n # Negative deltas are floored at 0.\n v3 = SiteReport.calculate_assets_started(\n previous_assets_total=107,\n previous_assets_not_started=92,\n current_assets_total=100,\n current_assets_not_started=90,\n )\n self.assertEqual(v3, 0)\n\n def test_series_navigation_and_sums(self):\n # Site-wide TOTAL series snapshots across three days\n d1 = self._aware(2024, 1, 10)\n d2 = self._aware(2024, 1, 20)\n d3 = self._aware(2024, 1, 31)\n\n r1 = self._mk_sr(\n dt=d1,\n report_name=SiteReport.ReportName.TOTAL,\n assets_started=3,\n )\n r2 = self._mk_sr(\n dt=d2,\n report_name=SiteReport.ReportName.TOTAL,\n assets_started=7,\n )\n r3 = self._mk_sr(\n dt=d3,\n report_name=SiteReport.ReportName.TOTAL,\n assets_started=10,\n )\n\n prev = SiteReport.objects.previous_in_series(\n report_name=SiteReport.ReportName.TOTAL,\n before=self._aware(2024, 1, 25),\n )\n self.assertEqual(prev.pk, r2.pk)\n\n # last_on_or_before_date_for_series\n last = SiteReport.objects.last_on_or_before_date_for_series(\n report_name=SiteReport.ReportName.TOTAL,\n on_or_before_date=date(2024, 1, 31),\n )\n self.assertEqual(last.pk, r3.pk)\n\n first = SiteReport.objects.first_on_or_after_date_for_series(\n report_name=SiteReport.ReportName.TOTAL,\n on_or_after_date=date(2024, 1, 15),\n on_or_before_date=date(2024, 1, 29),\n )\n self.assertEqual(first.pk, r2.pk)\n\n self.assertEqual(r2.previous_in_series().pk, r1.pk)\n self.assertEqual(r2.next_in_series().pk, r3.pk)\n\n summed = SiteReport.objects.sum_assets_started_for_series_between_dates(\n report_name=SiteReport.ReportName.TOTAL,\n start_date=date(2024, 1, 1),\n end_date=date(2024, 1, 31),\n )\n self.assertEqual(summed, 3 + 7 + 10)\n\n def test_per_campaign_and_topic_series_filters(self):\n camp = Campaign.objects.create(title=\"C1\", slug=\"c1\")\n # Per-campaign series\n d1 = self._aware(2023, 12, 1)\n d2 = self._aware(2023, 12, 2)\n s1 = self._mk_sr(dt=d1, campaign=camp, assets_total=1)\n s2 = self._mk_sr(dt=d2, campaign=camp, assets_total=2)\n\n prev = SiteReport.objects.previous_for_instance(s2)\n nxt = SiteReport.objects.next_for_instance(s1)\n self.assertEqual(prev.pk, s1.pk)\n self.assertEqual(nxt.pk, s2.pk)\n\n # Unspecified series (fallback), ensure no crash and no result\n none_prev = SiteReport.objects.previous_in_series()\n self.assertIsNone(none_prev)\n\n def test__series_filter_campaign_branch(self):\n camp = Campaign.objects.create(title=\"C\", slug=\"c\")\n # Two rows in same per-campaign series\n s1 = SiteReport.objects.create(campaign=camp, assets_total=1)\n s2 = SiteReport.objects.create(campaign=camp, assets_total=2)\n\n # Force a deterministic order\n SiteReport.objects.filter(pk=s1.pk).update(created_on=self._aware(2024, 1, 1))\n SiteReport.objects.filter(pk=s2.pk).update(created_on=self._aware(2024, 1, 2))\n\n prev = SiteReport.objects.previous_in_series(\n campaign=camp, before=self._aware(2024, 1, 3)\n )\n self.assertEqual(prev.pk, s2.pk)\n\n # And last_on_or_before path also using campaign filter\n last = SiteReport.objects.last_on_or_before_date_for_series(\n campaign=camp, on_or_before_date=date(2024, 1, 2)\n )\n self.assertEqual(last.pk, s2.pk)\n\n def test__series_filter_topic_branch(self):\n topic = Topic.objects.create(title=\"T\", slug=\"t\")\n s1 = SiteReport.objects.create(topic=topic, assets_total=1)\n s2 = SiteReport.objects.create(topic=topic, assets_total=2)\n SiteReport.objects.filter(pk=s1.pk).update(created_on=self._aware(2024, 2, 1))\n SiteReport.objects.filter(pk=s2.pk).update(created_on=self._aware(2024, 2, 2))\n\n first = SiteReport.objects.first_on_or_after_date_for_series(\n topic=topic,\n on_or_after_date=date(2024, 2, 1),\n on_or_before_date=date(2024, 2, 5),\n )\n self.assertEqual(first.pk, s1.pk)\n\n def test_series_filter_for_instance_topic_branch(self):\n topic = Topic.objects.create(title=\"T2\", slug=\"t2\")\n a = SiteReport.objects.create(topic=topic, assets_total=10)\n b = SiteReport.objects.create(topic=topic, assets_total=20)\n\n SiteReport.objects.filter(pk=a.pk).update(created_on=self._aware(2024, 3, 1))\n SiteReport.objects.filter(pk=b.pk).update(created_on=self._aware(2024, 3, 2))\n\n # IMPORTANT: refresh to pick up the updated created_on values\n a.refresh_from_db()\n b.refresh_from_db()\n\n self.assertEqual(b.previous_in_series().pk, a.pk)\n self.assertEqual(a.next_in_series().pk, b.pk)\n\n def test_series_filter_for_instance_retired_and_fallback(self):\n r = SiteReport.objects.create(\n report_name=SiteReport.ReportName.RETIRED_TOTAL, assets_total=1\n )\n # With only a single row, previous/next resolve via the RETIRED series Q()\n self.assertIsNone(r.previous_in_series())\n self.assertIsNone(r.next_in_series())\n\n blank = SiteReport.objects.create(assets_total=3) # report_name=\"\"\n self.assertIsNone(blank.previous_in_series())\n self.assertIsNone(blank.next_in_series())\n\n def test_to_debug_dict_includes_related_fields_and_counters(self):\n camp = Campaign.objects.create(title=\"CTitle\", slug=\"cslug\")\n topic = Topic.objects.create(title=\"TTitle\", slug=\"tslug\")\n\n sr_campaign = SiteReport.objects.create(\n campaign=camp, assets_total=9, assets_published=3\n )\n sr_topic = SiteReport.objects.create(\n topic=topic, items_published=4, items_unpublished=1\n )\n\n d1 = sr_campaign.to_debug_dict()\n self.assertIn(\"campaign\", d1)\n self.assertEqual(d1[\"campaign\"][\"id\"], camp.id)\n self.assertEqual(d1[\"campaign\"][\"title\"], \"CTitle\")\n self.assertEqual(d1[\"campaign\"][\"slug\"], \"cslug\")\n self.assertIn(\"counters\", d1)\n self.assertEqual(d1[\"counters\"][\"assets_total\"], 9)\n self.assertEqual(d1[\"counters\"][\"assets_published\"], 3)\n\n d2 = sr_topic.to_debug_dict()\n self.assertIn(\"topic\", d2)\n self.assertEqual(d2[\"topic\"][\"id\"], topic.id)\n self.assertEqual(d2[\"topic\"][\"title\"], \"TTitle\")\n self.assertEqual(d2[\"topic\"][\"slug\"], \"tslug\")\n self.assertEqual(d2[\"counters\"][\"items_published\"], 4)\n self.assertEqual(d2[\"counters\"][\"items_unpublished\"], 1)\n\n def test_first_on_or_after_with_upper_bound_campaign(self):\n camp = Campaign.objects.create(title=\"C-Bound\", slug=\"c-bound\")\n a = SiteReport.objects.create(campaign=camp)\n b = SiteReport.objects.create(campaign=camp)\n SiteReport.objects.filter(pk=a.pk).update(created_on=self._aware(2024, 5, 1))\n SiteReport.objects.filter(pk=b.pk).update(created_on=self._aware(2024, 5, 2))\n\n out = SiteReport.objects.first_on_or_after_date_for_series(\n campaign=camp,\n on_or_after_date=date(2024, 5, 1),\n on_or_before_date=date(2024, 5, 1),\n )\n self.assertEqual(out.pk, a.pk)\n\n def test_previous_in_series_defaults_to_now(self):\n r1 = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n r2 = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=r1.pk).update(created_on=self._aware(2024, 1, 10))\n SiteReport.objects.filter(pk=r2.pk).update(created_on=self._aware(2024, 1, 20))\n out = SiteReport.objects.previous_in_series(\n report_name=SiteReport.ReportName.TOTAL\n )\n self.assertEqual(out.pk, r2.pk)\n\n def test_sum_assets_started_treats_null_as_zero(self):\n r1 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL, assets_started=None\n )\n r2 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL, assets_started=None\n )\n SiteReport.objects.filter(pk=r1.pk).update(created_on=self._aware(2024, 2, 1))\n SiteReport.objects.filter(pk=r2.pk).update(created_on=self._aware(2024, 2, 2))\n total = SiteReport.objects.sum_assets_started_for_series_between_dates(\n report_name=SiteReport.ReportName.TOTAL,\n start_date=date(2024, 2, 1),\n end_date=date(2024, 2, 28),\n )\n self.assertEqual(total, 0)\n\n def test_to_debug_dict_campaign_status_and_topic_loop(self):\n camp = Campaign.objects.create(\n title=\"Camp\", slug=\"camp\"\n ) # status has a default\n topic = Topic.objects.create(title=\"Top\", slug=\"top\")\n\n sr_campaign = SiteReport.objects.create(campaign=camp, assets_total=1)\n sr_topic = SiteReport.objects.create(topic=topic, assets_total=2)\n\n d1 = sr_campaign.to_debug_dict()\n self.assertIn(\"campaign\", d1)\n # ensure the loop includes all three fields, including status\n self.assertEqual(d1[\"campaign\"][\"title\"], \"Camp\")\n self.assertEqual(d1[\"campaign\"][\"slug\"], \"camp\")\n self.assertIn(\"status\", d1[\"campaign\"])\n\n d2 = sr_topic.to_debug_dict()\n self.assertIn(\"topic\", d2)\n # ensure the loop includes both fields for topic\n self.assertEqual(d2[\"topic\"][\"title\"], \"Top\")\n self.assertEqual(d2[\"topic\"][\"slug\"], \"top\")\n\n def test_to_debug_json_serializes_and_includes_counters(self):\n camp = Campaign.objects.create(title=\"CJ\", slug=\"cj\")\n sr = SiteReport.objects.create(\n campaign=camp, assets_total=4, assets_published=2\n )\n out = sr.to_debug_json()\n parsed = json.loads(out)\n\n # basic shape checks\n self.assertIn(\"created_on\", parsed) # ISO string\n self.assertEqual(parsed[\"report_name\"], \"\")\n self.assertEqual(parsed[\"campaign\"][\"id\"], camp.id)\n\n # counters included and numeric values preserved\n self.assertEqual(parsed[\"counters\"][\"assets_total\"], 4)\n self.assertEqual(parsed[\"counters\"][\"assets_published\"], 2)\n\n def test_first_on_or_after_without_upper_bound_topic(self):\n # Create two topic reports; query without an upper bound s\n # hould still return the earliest on/after.\n topic = Topic.objects.create(title=\"UBT\", slug=\"ubt\")\n s1 = SiteReport.objects.create(topic=topic)\n s2 = SiteReport.objects.create(topic=topic)\n SiteReport.objects.filter(pk=s1.pk).update(created_on=self._aware(2024, 6, 1))\n SiteReport.objects.filter(pk=s2.pk).update(created_on=self._aware(2024, 6, 2))\n\n out = SiteReport.objects.first_on_or_after_date_for_series(\n topic=topic,\n on_or_after_date=date(2024, 6, 2),\n # no on_or_before_date here on purpose\n )\n self.assertEqual(out.pk, s2.pk)\n\n def test_to_debug_dict_skips_none_campaign_attrs(self):\n # Force the related-object cache to a stub that lacks some attrs\n from types import SimpleNamespace\n\n camp = Campaign.objects.create(title=\"C\", slug=\"c\")\n sr = SiteReport.objects.create(campaign=camp, assets_total=1)\n\n # Populate fields_cache so descriptor returns this stub instead of hitting DB\n sr._state.fields_cache[\"campaign\"] = SimpleNamespace(title=\"OnlyTitle\")\n d = sr.to_debug_dict()\n\n self.assertIn(\"campaign\", d)\n self.assertEqual(d[\"campaign\"][\"id\"], camp.id)\n # title present, slug/status omitted because getattr(...) returned None\n self.assertEqual(d[\"campaign\"][\"title\"], \"OnlyTitle\")\n self.assertNotIn(\"slug\", d[\"campaign\"])\n self.assertNotIn(\"status\", d[\"campaign\"])\n\n def test_to_debug_dict_skips_none_topic_attrs(self):\n # Force the related-object cache to a stub that lacks one of the looped attrs\n from types import SimpleNamespace\n\n t = Topic.objects.create(title=\"TT\", slug=\"tt\")\n sr = SiteReport.objects.create(topic=t, assets_total=2)\n\n sr._state.fields_cache[\"topic\"] = SimpleNamespace(slug=\"only-slug\")\n d = sr.to_debug_dict()\n\n self.assertIn(\"topic\", d)\n self.assertEqual(d[\"topic\"][\"id\"], t.id)\n # slug present, title omitted because getattr(...) returned None\n self.assertEqual(d[\"topic\"][\"slug\"], \"only-slug\")\n self.assertNotIn(\"title\", d[\"topic\"])\n\n\nclass KeyMetricsReportTestCase(TestCase):\n def _aware(self, y, m, d, hh=12, mm=0, ss=0):\n tz = timezone.get_current_timezone()\n return timezone.make_aware(datetime(y, m, d, hh, mm, ss), tz)\n\n def _mk_sr(self, dt, report_name, **counters):\n sr = SiteReport.objects.create(\n report_name=report_name,\n **counters,\n )\n SiteReport.objects.filter(pk=sr.pk).update(created_on=dt)\n return SiteReport.objects.get(pk=sr.pk)\n\n def test_helpers(self):\n # FY math\n self.assertEqual(\n KeyMetricsReport.get_fiscal_year_for_date(date(2023, 10, 1)),\n 2024,\n )\n self.assertEqual(\n KeyMetricsReport.get_fiscal_year_for_date(date(2024, 9, 30)),\n 2024,\n )\n self.assertEqual(\n KeyMetricsReport.get_fiscal_quarter_for_date(date(2024, 2, 1)),\n 2,\n )\n self.assertEqual(\n KeyMetricsReport.get_fiscal_quarter_for_date(date(2024, 10, 1)),\n 1,\n )\n # Month bounds (leap year Feb)\n first, last = KeyMetricsReport.month_bounds(date(2024, 2, 10))\n self.assertEqual(first, date(2024, 2, 1))\n self.assertEqual(last, date(2024, 2, 29))\n\n def test_upsert_month_from_sitereports(self):\n # Baselines at 2023-12-31; EOM at 2024-01-31\n base_dt = self._aware(2023, 12, 31, 9, 0, 0)\n eom_dt = self._aware(2024, 1, 31, 23, 0, 0)\n\n # TOTAL baseline + EOM\n self._mk_sr(\n base_dt,\n SiteReport.ReportName.TOTAL,\n assets_published=100,\n assets_completed=50,\n users_activated=10,\n anonymous_transcriptions=5,\n transcriptions_saved=20,\n tag_uses=40,\n )\n self._mk_sr(\n eom_dt,\n SiteReport.ReportName.TOTAL,\n assets_published=130,\n assets_completed=70,\n users_activated=16,\n anonymous_transcriptions=8,\n transcriptions_saved=26,\n tag_uses=50,\n )\n\n # RETIRED_TOTAL baseline + EOM\n self._mk_sr(\n base_dt,\n SiteReport.ReportName.RETIRED_TOTAL,\n assets_published=10,\n assets_completed=5,\n users_activated=1,\n anonymous_transcriptions=2,\n transcriptions_saved=3,\n tag_uses=4,\n )\n self._mk_sr(\n eom_dt,\n SiteReport.ReportName.RETIRED_TOTAL,\n assets_published=15,\n assets_completed=8,\n users_activated=2,\n anonymous_transcriptions=3,\n transcriptions_saved=5,\n tag_uses=6,\n )\n\n # Daily assets_started within the month (sums to 15 + 5 = 20)\n self._mk_sr(\n self._aware(2024, 1, 10),\n SiteReport.ReportName.TOTAL,\n assets_started=10,\n )\n self._mk_sr(\n self._aware(2024, 1, 20),\n SiteReport.ReportName.TOTAL,\n assets_started=5,\n )\n self._mk_sr(\n self._aware(2024, 1, 11),\n SiteReport.ReportName.RETIRED_TOTAL,\n assets_started=3,\n )\n self._mk_sr(\n self._aware(2024, 1, 21),\n SiteReport.ReportName.RETIRED_TOTAL,\n assets_started=2,\n )\n\n # Upsert month\n m = KeyMetricsReport.upsert_month(year=2024, month=1)\n self.assertIsNotNone(m)\n self.assertEqual(m.fiscal_year, 2024)\n self.assertEqual(m.fiscal_quarter, 2)\n self.assertEqual(m.month, 1)\n\n # Deltas: see analysis; expect 35, 23, 7, 4, 8, 12 and started=20\n self.assertEqual(m.assets_published, 35)\n self.assertEqual(m.assets_completed, 23)\n self.assertEqual(m.users_activated, 7)\n self.assertEqual(m.anonymous_transcriptions, 4)\n self.assertEqual(m.transcriptions_saved, 8)\n self.assertEqual(m.tag_uses, 12)\n self.assertEqual(m.assets_started, 20)\n\n # __str__ + filenames\n self.assertIn(\"FY2024M01\", str(m))\n self.assertTrue(m.csv_filename().startswith(\"key_metrics_monthly_fy2024\"))\n\n def test_upsert_quarter_and_fiscal_year_rollups(self):\n # Create monthly rows for FY2024 Q2 (Jan & Feb present)\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 1, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n month=1,\n assets_published=10,\n assets_started=2,\n assets_completed=3,\n users_activated=5,\n anonymous_transcriptions=7,\n transcriptions_saved=11,\n tag_uses=13,\n crowd_visits=None,\n avg_visit_seconds=Decimal(\"10.50\"),\n )\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 2, 1),\n period_end=date(2024, 2, 29),\n fiscal_year=2024,\n fiscal_quarter=2,\n month=2,\n assets_published=20,\n assets_started=3,\n assets_completed=4,\n users_activated=6,\n anonymous_transcriptions=8,\n transcriptions_saved=12,\n tag_uses=14,\n # manual present in Feb only\n crowd_visits=100,\n avg_visit_seconds=None,\n )\n\n # Quarter upsert should sum calc fields; manual sums only when present\n q2 = KeyMetricsReport.upsert_quarter(fiscal_year=2024, fiscal_quarter=2)\n self.assertIsNotNone(q2)\n self.assertEqual(q2.assets_published, 30)\n self.assertEqual(q2.assets_started, 5)\n self.assertEqual(q2.assets_completed, 7)\n self.assertEqual(q2.users_activated, 11)\n self.assertEqual(q2.anonymous_transcriptions, 15)\n self.assertEqual(q2.transcriptions_saved, 23)\n self.assertEqual(q2.tag_uses, 27)\n # Manual: only Feb had a value, so total=100, avg from Jan only\n self.assertEqual(q2.crowd_visits, 100)\n self.assertEqual(q2.avg_visit_seconds, Decimal(\"10.50\"))\n\n # Fiscal year rollup on FY2024 should equal Jan+Feb (for now)\n fy = KeyMetricsReport.upsert_fiscal_year(fiscal_year=2024)\n self.assertIsNotNone(fy)\n self.assertEqual(fy.assets_published, 30)\n self.assertEqual(fy.crowd_visits, 100)\n self.assertEqual(fy.avg_visit_seconds, Decimal(\"10.50\"))\n\n # String and filenames\n self.assertIn(\"FY2024 Q2\", str(q2))\n self.assertTrue(q2.csv_filename().startswith(\"key_metrics_quarterly_fy2024\"))\n self.assertIn(\"FY2024 Report\", str(fy))\n self.assertTrue(fy.csv_filename().startswith(\"key_metrics_fiscal_year_fy2024\"))\n\n def test___str___fallback_when_fields_incomplete(self):\n # QUARTERLY without fiscal_quarter so fallback label path\n q = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 4, 1),\n period_end=date(2024, 6, 30),\n fiscal_year=2024,\n fiscal_quarter=None,\n )\n s = str(q)\n self.assertIn(\"KeyMetricsReport QUARTERLY\", s)\n self.assertIn(\"2024-04-01\", s)\n self.assertIn(\"2024-06-30\", s)\n\n # MONTHLY without month so fallback label path\n m = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 5, 1),\n period_end=date(2024, 5, 31),\n fiscal_year=2024,\n month=None,\n )\n s2 = str(m)\n self.assertIn(\"KeyMetricsReport MONTHLY\", s2)\n\n def test_quarter_helper_edges(self):\n # Q3 and Q4 branches\n self.assertEqual(\n KeyMetricsReport.get_fiscal_quarter_for_date(date(2024, 4, 1)), 3\n )\n self.assertEqual(\n KeyMetricsReport.get_fiscal_quarter_for_date(date(2024, 7, 1)), 4\n )\n\n def test__format_value_for_csv_variants(self):\n rep = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,\n period_start=date(2023, 10, 1),\n period_end=date(2024, 9, 30),\n fiscal_year=2024,\n )\n\n self.assertEqual(rep._format_value_for_csv(\"crowd_visits\", None), \"\")\n\n # Manual Decimal (avg_visit_seconds) to string with 2 decimals\n self.assertEqual(\n rep._format_value_for_csv(\"avg_visit_seconds\", Decimal(\"10\")),\n \"10.00\",\n )\n\n self.assertEqual(rep._format_value_for_csv(\"crowd_visits\", 0), 0)\n\n self.assertEqual(rep._format_value_for_csv(\"assets_started\", None), 0)\n\n # Unknown field fallback: None to \"\", non-None to value passthrough\n self.assertEqual(rep._format_value_for_csv(\"unknown_field\", None), \"\")\n self.assertEqual(rep._format_value_for_csv(\"unknown_field\", \"x\"), \"x\")\n\n def test_upsert_month_returns_none_when_no_snapshots(self):\n out = KeyMetricsReport.upsert_month(year=2025, month=6)\n self.assertIsNone(out)\n\n def test_upsert_quarter_invalid_quarter_raises(self):\n with self.assertRaises(ValueError):\n KeyMetricsReport.upsert_quarter(fiscal_year=2024, fiscal_quarter=5)\n\n def test_quarter_month_specs_all_quarters(self):\n # Q1\n q1 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2023, 10, 1),\n period_end=date(2023, 12, 31),\n fiscal_year=2024,\n fiscal_quarter=1,\n )\n self.assertEqual(\n q1._quarter_month_specs(), [(2023, 10), (2023, 11), (2023, 12)]\n )\n\n # Q3\n q3 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 4, 1),\n period_end=date(2024, 6, 30),\n fiscal_year=2024,\n fiscal_quarter=3,\n )\n self.assertEqual(q3._quarter_month_specs(), [(2024, 4), (2024, 5), (2024, 6)])\n\n # Q4\n q4 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 7, 1),\n period_end=date(2024, 9, 30),\n fiscal_year=2024,\n fiscal_quarter=4,\n )\n self.assertEqual(q4._quarter_month_specs(), [(2024, 7), (2024, 8), (2024, 9)])\n\n def test_month_bounds_handles_december(self):\n first, last = KeyMetricsReport.month_bounds(date(2024, 12, 10))\n self.assertEqual(first, date(2024, 12, 1))\n self.assertEqual(last, date(2024, 12, 31))\n\n def test__monthly_from_sitereports_returns_empty_dict_when_no_eom(self):\n vals = KeyMetricsReport._monthly_from_sitereports(\n month_start=date(2030, 5, 1),\n month_end=date(2030, 5, 31),\n )\n self.assertEqual(vals, {}) # no snapshots at all\n\n def test__monthly_from_sitereports_baseline_fallback_inside_month(self):\n # No snapshots before month start; first snapshot inside the month\n start = date(2024, 3, 1)\n end = date(2024, 3, 31)\n\n # TOTAL: baseline inside month (10) -> EOM (15)\n self._mk_sr(\n self._aware(2024, 3, 5, 9, 0, 0),\n SiteReport.ReportName.TOTAL,\n assets_published=10,\n )\n self._mk_sr(\n self._aware(2024, 3, 31, 23, 0, 0),\n SiteReport.ReportName.TOTAL,\n assets_published=15,\n )\n\n # RETIRED: baseline inside month (4) -> EOM (7)\n self._mk_sr(\n self._aware(2024, 3, 10, 9, 0, 0),\n SiteReport.ReportName.RETIRED_TOTAL,\n assets_published=4,\n )\n self._mk_sr(\n self._aware(2024, 3, 31, 23, 0, 0),\n SiteReport.ReportName.RETIRED_TOTAL,\n assets_published=7,\n )\n\n vals = KeyMetricsReport._monthly_from_sitereports(\n month_start=start, month_end=end\n )\n # delta should be (15+7) - (10+4) = 8\n self.assertEqual(vals[\"assets_published\"], 8)\n\n def test__monthly_from_sitereports_treats_missing_series_as_zero(self):\n # Only TOTAL snapshots; RETIRED series absent\n start = date(2024, 4, 1)\n end = date(2024, 4, 30)\n\n # baseline inside month (100) -> EOM (110)\n self._mk_sr(\n self._aware(2024, 4, 5, 9, 0, 0),\n SiteReport.ReportName.TOTAL,\n assets_published=100,\n )\n self._mk_sr(\n self._aware(2024, 4, 30, 23, 0, 0),\n SiteReport.ReportName.TOTAL,\n assets_published=110,\n )\n\n vals = KeyMetricsReport._monthly_from_sitereports(\n month_start=start, month_end=end\n )\n # RETIRED contributes 0 via the helper that treats None as 0\n self.assertEqual(vals[\"assets_published\"], 10)\n\n def test_upsert_quarter_returns_none_when_no_monthlies_all_quarters(self):\n # Q1\n out1 = KeyMetricsReport.upsert_quarter(fiscal_year=2027, fiscal_quarter=1)\n self.assertIsNone(out1)\n # Q3\n out3 = KeyMetricsReport.upsert_quarter(fiscal_year=2027, fiscal_quarter=3)\n self.assertIsNone(out3)\n # Q4\n out4 = KeyMetricsReport.upsert_quarter(fiscal_year=2027, fiscal_quarter=4)\n self.assertIsNone(out4)\n\n def test_upsert_fiscal_year_returns_none_when_no_monthlies(self):\n out = KeyMetricsReport.upsert_fiscal_year(fiscal_year=2029)\n self.assertIsNone(out)\n\n def test__calendar_year_for_month_in_fy_helper(self):\n rep = KeyMetricsReport(\n period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,\n period_start=date(2023, 10, 1),\n period_end=date(2024, 9, 30),\n fiscal_year=2024,\n )\n # Oct in FY should map to previous calendar year\n self.assertEqual(rep._calendar_year_for_month_in_fy(10, 2024), 2023)\n # Jun in FY maps to the FY year\n self.assertEqual(rep._calendar_year_for_month_in_fy(6, 2024), 2024)\n\n def test_quarter_month_specs_q2(self):\n q2 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n self.assertEqual(q2._quarter_month_specs(), [(2024, 1), (2024, 2), (2024, 3)])\n\n\nclass KeyMetricsReportCsvTestCase(TestCase):\n def setUp(self):\n # FY2023 FY row (for lifetime math)\n self.fy23 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,\n period_start=date(2022, 10, 1),\n period_end=date(2023, 9, 30),\n fiscal_year=2023,\n assets_published=50,\n assets_started=5,\n assets_completed=7,\n users_activated=11,\n anonymous_transcriptions=13,\n transcriptions_saved=17,\n tag_uses=19,\n crowd_visits=30,\n avg_visit_seconds=Decimal(\"9.00\"),\n )\n\n # FY2024 Q1 (for quarterly lifetime math on Q2)\n self.q1_24 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2023, 10, 1),\n period_end=date(2023, 12, 31),\n fiscal_year=2024,\n fiscal_quarter=1,\n assets_published=7,\n assets_started=1,\n assets_completed=2,\n users_activated=3,\n anonymous_transcriptions=4,\n transcriptions_saved=5,\n tag_uses=6,\n crowd_visits=None,\n avg_visit_seconds=Decimal(\"8.00\"),\n )\n\n # FY2024 monthly rows for Q2: Jan, Feb present only\n self.jan24 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 1, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n month=1,\n assets_published=10,\n assets_started=2,\n assets_completed=3,\n users_activated=5,\n anonymous_transcriptions=7,\n transcriptions_saved=11,\n tag_uses=13,\n crowd_visits=None,\n )\n self.feb24 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 2, 1),\n period_end=date(2024, 2, 29),\n fiscal_year=2024,\n fiscal_quarter=2,\n month=2,\n assets_published=20,\n assets_started=3,\n assets_completed=4,\n users_activated=6,\n anonymous_transcriptions=8,\n transcriptions_saved=12,\n tag_uses=14,\n crowd_visits=100,\n )\n\n # Upsert Q2 and FY2024 so we can render CSVs with proper totals\n self.q2_24 = KeyMetricsReport.upsert_quarter(\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n self.fy24 = KeyMetricsReport.upsert_fiscal_year(fiscal_year=2024)\n\n def _csv_as_lines(self, rep: KeyMetricsReport) -> list[list[str]]:\n raw = rep.render_csv().decode(\"utf-8\")\n return [line.split(\",\") for line in raw.strip().splitlines()]\n\n def test_monthly_csv_headers_and_values(self):\n # Build a synthetic single-month report to test header label only\n m = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 6, 1),\n period_end=date(2024, 6, 30),\n fiscal_year=2024,\n fiscal_quarter=3,\n month=6,\n assets_published=1,\n )\n lines = self._csv_as_lines(m)\n # Header: \"Metric\", \"
\"\n self.assertEqual(lines[0][0], \"Metric\")\n self.assertEqual(lines[0][1], \"June\")\n\n # One known metric row check\n labels = [row[0] for row in lines[1:]]\n vals = [row[1] for row in lines[1:]]\n pub_idx = labels.index(\"Assets published\")\n self.assertEqual(int(vals[pub_idx]), 1)\n\n def test_quarterly_csv_headers_totals_and_lifetime(self):\n lines = self._csv_as_lines(self.q2_24)\n header = lines[0]\n\n # Months present (Jan, Feb), then \"FY24 Q2 totals\", \"FY24 Lifetime totals\"\n self.assertEqual(header[0], \"Metric\")\n self.assertIn(\"January\", header)\n self.assertIn(\"February\", header)\n self.assertIn(\"FY24 Q2 totals\", header)\n self.assertIn(\"FY24 Lifetime totals\", header)\n\n # Assets published row:\n # Jan(10), Feb(20) => quarter total=30\n # Lifetime = FY2023 FY(50) + FY2024 Q1(7) = 57\n labels = [row[0] for row in lines[1:]]\n ap_idx = labels.index(\"Assets published\")\n row = lines[1 + ap_idx]\n # [label, Jan, Feb, Q2 total, Lifetime]\n self.assertEqual(int(row[1]), 10)\n self.assertEqual(int(row[2]), 20)\n self.assertEqual(int(row[3]), 30)\n self.assertEqual(int(row[4]), 57)\n\n # Manual example (Crowd.loc.gov visits):\n # Jan(None), Feb(100) => Q2 total=100 (not blank)\n # Lifetime = FY2023 FY(30) + Q1(None) => 30\n cv_idx = labels.index(\"Crowd.loc.gov visits\")\n row2 = lines[1 + cv_idx]\n self.assertEqual(row2[1], \"\") # January empty\n self.assertEqual(int(row2[2]), 100)\n self.assertEqual(int(row2[3]), 100)\n self.assertEqual(int(row2[4]), 30)\n\n def test_fiscal_year_csv_headers_totals_and_lifetime(self):\n # Ensure FY rows exist for lifetime (FY2023 and FY2024 already present)\n lines = self._csv_as_lines(self.fy24)\n header = lines[0]\n\n # Header pattern:\n # Metric | (FY24 Q1 totals if present) | Q2 totals | Q3 totals? | Q4 totals?\n # | FY24 totals | FY24 Lifetime totals\n self.assertEqual(header[0], \"Metric\")\n self.assertIn(\"FY24 Q1 totals\", header)\n self.assertIn(\"Q2 totals\", header)\n self.assertIn(\"FY24 totals\", header)\n self.assertIn(\"FY24 Lifetime totals\", header)\n\n labels = [row[0] for row in lines[1:]]\n ap_idx = labels.index(\"Assets published\")\n row = lines[1 + ap_idx]\n\n # With our setup:\n # Q1 assets_published=7 (preset), Q2=30 (from jan+feb),\n # year total = 37, lifetime = FY2023 FY(50) + FY2024 FY(37) = 87\n # Header columns could be: Metric, FY24 Q1 totals, Q2 totals,\n # FY24 totals, FY24 Lifetime totals (Q3/Q4 absent)\n # Find indices dynamically.\n h = header\n q1_i = h.index(\"FY24 Q1 totals\")\n q2_i = h.index(\"Q2 totals\")\n yt_i = h.index(\"FY24 totals\")\n lt_i = h.index(\"FY24 Lifetime totals\")\n\n self.assertEqual(int(row[q1_i]), 7)\n self.assertEqual(int(row[q2_i]), 30)\n self.assertEqual(int(row[yt_i]), 37)\n expected_lifetime = self.fy23.assets_published + self.fy24.assets_published\n self.assertEqual(int(row[lt_i]), expected_lifetime)\n\n def test_str_formats(self):\n # Monthly string covers Oct (calendar year is fy-1)\n oct_row = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2023, 10, 1),\n period_end=date(2023, 10, 31),\n fiscal_year=2024,\n fiscal_quarter=1,\n month=10,\n )\n s = str(oct_row)\n self.assertIn(\"FY2024M10\", s)\n self.assertIn(\"(October 2023)\", s)\n\n def test_quarterly_csv_when_no_monthlies_and_no_priors(self):\n # Create a standalone quarterly row with no monthly rows in that quarter\n q = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2019, 10, 1),\n period_end=date(2019, 12, 31),\n fiscal_year=2020,\n fiscal_quarter=1,\n )\n lines = self._csv_as_lines(q)\n\n # Header should be: Metric | FY25 Q3 totals | FY25 Lifetime totals\n self.assertEqual(lines[0][0], \"Metric\")\n self.assertIn(\"FY20 Q1 totals\", lines[0])\n self.assertIn(\"FY20 Lifetime totals\", lines[0])\n self.assertEqual(len(lines[0]), 3)\n\n labels = [r[0] for r in lines[1:]]\n # Calculated field: totals are numeric, lifetime is 0\n ap_i = labels.index(\"Assets published\")\n ap_row = lines[1 + ap_i]\n self.assertEqual(int(ap_row[1]), 0) # quarter total\n self.assertEqual(int(ap_row[2]), 0) # lifetime total\n\n # Manual field: totals should be blank when no values\n cv_i = labels.index(\"Crowd.loc.gov visits\")\n cv_row = lines[1 + cv_i]\n self.assertEqual(cv_row[1], \"\") # quarter total blank\n self.assertEqual(cv_row[2], \"\") # lifetime total blank\n\n def test_fiscal_year_csv_headers_when_q1_missing(self):\n # Create an FY row and only Q2 and Q4 quarters for that FY\n fy = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,\n period_start=date(2025, 10, 1),\n period_end=date(2026, 9, 30),\n fiscal_year=2026,\n )\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2026, 1, 1),\n period_end=date(2026, 3, 31),\n fiscal_year=2026,\n fiscal_quarter=2,\n assets_published=12,\n )\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2026, 7, 1),\n period_end=date(2026, 9, 30),\n fiscal_year=2026,\n fiscal_quarter=4,\n assets_published=8,\n )\n\n lines = self._csv_as_lines(fy)\n header = lines[0]\n\n self.assertEqual(header[0], \"Metric\")\n self.assertNotIn(\"FY26 Q1 totals\", header)\n self.assertIn(\"Q2 totals\", header)\n self.assertNotIn(\"Q3 totals\", header)\n self.assertIn(\"Q4 totals\", header)\n self.assertIn(\"FY26 totals\", header)\n self.assertIn(\"FY26 Lifetime totals\", header)\n\n labels = [r[0] for r in lines[1:]]\n ap_i = labels.index(\"Assets published\")\n row = lines[1 + ap_i]\n\n q2_i = header.index(\"Q2 totals\")\n q4_i = header.index(\"Q4 totals\")\n yt_i = header.index(\"FY26 totals\")\n lt_i = header.index(\"FY26 Lifetime totals\")\n\n self.assertEqual(int(row[q2_i]), 12)\n self.assertEqual(int(row[q4_i]), 8)\n self.assertEqual(int(row[yt_i]), 20)\n\n # Lifetime sums all FY rows <= 2026 (FY2023 + FY2024 + FY2026)\n expected_lifetime = (\n self.fy23.assets_published + self.fy24.assets_published + 0\n ) # FY2026 FY row has no stored value in this test\n self.assertEqual(int(row[lt_i]), expected_lifetime)\n\n def test_format_value_for_csv_non_decimal_avg(self):\n rep = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,\n period_start=date(2025, 10, 1),\n period_end=date(2026, 9, 30),\n fiscal_year=2026,\n )\n self.assertEqual(\n rep._format_value_for_csv(\"avg_visit_seconds\", 12.3),\n \"12.3\",\n )\n"
},
{
"path": "concordia/tests/test_parser.py",
"content": "from types import SimpleNamespace\nfrom unittest import mock\n\nimport requests\nfrom django.test import TestCase\nfrom requests.models import Response\n\nimport concordia.parser as parser_mod\nfrom concordia.parser import extract_og_image, fetch_blog_posts, paginate_blog_posts\n\nTITLE = \"Whatβs New Online at the Library of Congress: May 2025\"\nLINK = \"https://blogs.loc.gov/thesignal/2025/05/new-loc-may-2025/\"\nRSS = \"\"\"\n\n \n The Signal \n - \n
%s \n \n - \n
Volunteers Leverage OCR to Transcribe Library of Congress Digit... \n \n \n \n \n \n \"\"\" % (\n TITLE,\n LINK,\n)\nIMAGE = \"https://blogs.loc.gov/thesignal/files/2025/05/loc-2017698702.png\"\nHTML = \"\"\"\n \n \n \n \n \"\"\" % (\n link1,\n link2,\n )\n mock_response = mock.MagicMock(spec=Response)\n mock_response.content = rss\n mock_response.status_code = 200\n mock_get.return_value = mock_response\n\n # run the celery task\n fetch_and_cache_blog_images()\n\n mock_extract.assert_any_call(link1)\n mock_extract.assert_any_call(link2)\n self.assertEqual(mock_extract.call_count, 2)\n\n @mock.patch(\"concordia.tasks.blog.extract_og_image\")\n @mock.patch(\"concordia.tasks.blog.fetch_blog_posts\")\n def test_skips_items_with_no_link(self, mock_fetch, mock_extract):\n # Provide one item without a link and one with a link to make\n # sure we handle no link correctly\n class DummyLink:\n def __init__(self, text):\n self.text = text\n\n class DummyItem:\n def __init__(self, link):\n self._link = link\n\n def find(self, name):\n return self._link if name == \"link\" else None\n\n item_no_link = DummyItem(None)\n item_with_link = DummyItem(DummyLink(\"https://example.invalid/post\"))\n mock_fetch.return_value = [item_no_link, item_with_link]\n\n fetch_and_cache_blog_images()\n\n mock_extract.assert_called_once_with(\"https://example.invalid/post\")\n"
},
{
"path": "concordia/tests/test_tasks_housekeeping.py",
"content": "from unittest import mock\n\nfrom django.test import TestCase\n\nfrom concordia.tasks.housekeeping import clear_sessions\n\n\nclass ClearSessionsTaskTests(TestCase):\n def test_calls_django_clearsessions_command(self):\n # Verify the task invokes Django's clearsessions management command.\n with mock.patch(\"concordia.tasks.housekeeping.call_command\") as mock_call:\n result = clear_sessions()\n self.assertIsNone(result)\n mock_call.assert_called_once_with(\"clearsessions\")\n\n def test_raises_when_call_command_fails(self):\n # Ensure exceptions from the management command propagate.\n with mock.patch(\n \"concordia.tasks.housekeeping.call_command\",\n side_effect=RuntimeError(\"boom\"),\n ):\n with self.assertRaises(RuntimeError):\n clear_sessions()\n"
},
{
"path": "concordia/tests/test_tasks_next_asset.py",
"content": "from unittest import mock\n\nfrom django.test import TestCase\nfrom django.utils import timezone\n\nfrom concordia.models import (\n NextReviewableCampaignAsset,\n NextReviewableTopicAsset,\n NextTranscribableCampaignAsset,\n NextTranscribableTopicAsset,\n TranscriptionStatus,\n)\nfrom concordia.tasks.next_asset.renew import renew_next_asset_cache\nfrom concordia.tasks.next_asset.reviewable import (\n clean_next_reviewable_for_campaign,\n clean_next_reviewable_for_topic,\n populate_next_reviewable_for_campaign,\n populate_next_reviewable_for_topic,\n)\nfrom concordia.tasks.next_asset.transcribable import (\n clean_next_transcribable_for_campaign,\n clean_next_transcribable_for_topic,\n populate_next_transcribable_for_campaign,\n populate_next_transcribable_for_topic,\n)\nfrom concordia.utils import get_anonymous_user\n\nfrom .utils import (\n CreateTestUsers,\n create_asset,\n create_topic,\n create_transcription,\n)\n\n\nclass PopulateNextAssetTasksTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anon = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(slug=\"test-asset-1\", title=\"Test Asset 1\")\n self.asset2 = create_asset(\n item=self.asset1.item, slug=\"test-asset-2\", title=\"Test Asset 2\"\n )\n self.topic = create_topic(project=self.asset1.item.project)\n self.campaign = self.asset1.campaign\n\n def test_populate_next_transcribable_for_campaign(self):\n populate_next_transcribable_for_campaign(campaign_id=self.campaign.id)\n self.assertEqual(\n NextTranscribableCampaignAsset.objects.filter(\n campaign=self.campaign\n ).count(),\n 2,\n )\n\n def test_populate_next_transcribable_for_topic(self):\n populate_next_transcribable_for_topic(topic_id=self.topic.id)\n self.assertEqual(\n NextTranscribableTopicAsset.objects.filter(topic=self.topic).count(), 2\n )\n\n def test_populate_next_reviewable_for_campaign(self):\n create_transcription(\n asset=self.asset1, user=self.anon, submitted=timezone.now()\n )\n create_transcription(\n asset=self.asset2, user=self.user, submitted=timezone.now()\n )\n populate_next_reviewable_for_campaign(campaign_id=self.campaign.id)\n self.assertEqual(\n NextReviewableCampaignAsset.objects.filter(campaign=self.campaign).count(),\n 2,\n )\n\n def test_populate_next_reviewable_for_topic(self):\n create_transcription(\n asset=self.asset1, user=self.anon, submitted=timezone.now()\n )\n create_transcription(\n asset=self.asset2, user=self.user, submitted=timezone.now()\n )\n populate_next_reviewable_for_topic(topic_id=self.topic.id)\n self.assertEqual(\n NextReviewableTopicAsset.objects.filter(topic=self.topic).count(), 2\n )\n\n @mock.patch(\"concordia.tasks.next_asset.transcribable.logger\")\n def test_populate_next_transcribable_for_campaign_missing(self, mock_logger):\n populate_next_transcribable_for_campaign(campaign_id=9999)\n mock_logger.error.assert_called_once()\n\n @mock.patch(\"concordia.tasks.next_asset.transcribable.logger\")\n def test_populate_next_transcribable_for_topic_missing(self, mock_logger):\n populate_next_transcribable_for_topic(topic_id=9999)\n mock_logger.error.assert_called_once()\n\n @mock.patch(\"concordia.tasks.next_asset.reviewable.logger\")\n def test_populate_next_reviewable_for_campaign_missing(self, mock_logger):\n populate_next_reviewable_for_campaign(campaign_id=9999)\n mock_logger.error.assert_called_once()\n\n @mock.patch(\"concordia.tasks.next_asset.reviewable.logger\")\n def test_populate_next_reviewable_for_topic_missing(self, mock_logger):\n populate_next_reviewable_for_topic(topic_id=9999)\n mock_logger.error.assert_called_once()\n\n @mock.patch(\"concordia.tasks.next_asset.transcribable.logger\")\n def test_populate_next_transcribable_for_campaign_none_needed(self, mock_logger):\n for i in range(3, 103):\n asset = create_asset(item=self.asset1.item, slug=f\"dummy-{i}\")\n NextTranscribableCampaignAsset.objects.create(\n asset=asset,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n campaign=self.campaign,\n sequence=asset.sequence,\n transcription_status=asset.transcription_status,\n )\n populate_next_transcribable_for_campaign(campaign_id=self.campaign.id)\n mock_logger.info.assert_any_call(\n \"Campaign %s already has %s next transcribable assets\", self.campaign, 100\n )\n\n @mock.patch(\"concordia.tasks.next_asset.transcribable.logger\")\n def test_populate_next_transcribable_for_topic_none_needed(self, mock_logger):\n for i in range(3, 103):\n asset = create_asset(item=self.asset1.item, slug=f\"dummy-{i}\")\n NextTranscribableTopicAsset.objects.create(\n asset=asset,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n topic=self.topic,\n sequence=asset.sequence,\n transcription_status=asset.transcription_status,\n )\n populate_next_transcribable_for_topic(topic_id=self.topic.id)\n mock_logger.info.assert_any_call(\n \"Topic %s already has %s next transcribable assets\", self.topic, 100\n )\n\n @mock.patch(\"concordia.tasks.next_asset.reviewable.logger\")\n def test_populate_next_reviewable_for_campaign_none_needed(self, mock_logger):\n create_transcription(\n asset=self.asset1, user=self.user, submitted=timezone.now()\n )\n for i in range(3, 103):\n asset = create_asset(item=self.asset1.item, slug=f\"r-{i}\")\n create_transcription(asset=asset, user=self.user, submitted=timezone.now())\n NextReviewableCampaignAsset.objects.create(\n asset=asset,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n campaign=self.campaign,\n sequence=asset.sequence,\n transcriber_ids=[self.user.id],\n )\n\n populate_next_reviewable_for_campaign(campaign_id=self.campaign.id)\n mock_logger.info.assert_any_call(\n \"Campaign %s already has %s next reviewable assets\", self.campaign, 100\n )\n\n @mock.patch(\"concordia.tasks.next_asset.reviewable.logger\")\n def test_populate_next_reviewable_for_topic_none_needed(self, mock_logger):\n create_transcription(\n asset=self.asset1, user=self.user, submitted=timezone.now()\n )\n for i in range(3, 103):\n asset = create_asset(item=self.asset1.item, slug=f\"t-{i}\")\n create_transcription(asset=asset, user=self.user, submitted=timezone.now())\n NextReviewableTopicAsset.objects.create(\n asset=asset,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n topic=self.topic,\n sequence=asset.sequence,\n transcriber_ids=[self.user.id],\n )\n\n populate_next_reviewable_for_topic(topic_id=self.topic.id)\n mock_logger.info.assert_any_call(\n \"Topic %s already has %s next reviewable assets\", self.topic, 100\n )\n\n @mock.patch(\"concordia.tasks.next_asset.reviewable.logger\")\n def test_populate_next_reviewable_for_campaign_none_found(self, mock_logger):\n create_transcription(\n asset=self.asset1, user=self.user, submitted=timezone.now()\n )\n\n NextReviewableCampaignAsset.objects.create(\n asset=self.asset1,\n item=self.asset1.item,\n item_item_id=self.asset1.item.item_id,\n project=self.asset1.item.project,\n project_slug=self.asset1.item.project.slug,\n campaign=self.campaign,\n sequence=self.asset1.sequence,\n transcriber_ids=[self.user.id],\n )\n\n populate_next_reviewable_for_campaign(campaign_id=self.campaign.id)\n mock_logger.info.assert_any_call(\n \"No reviewable assets found in campaign %s\", self.campaign\n )\n\n @mock.patch(\"concordia.tasks.next_asset.reviewable.logger\")\n def test_populate_next_reviewable_for_topic_none_found(self, mock_logger):\n create_transcription(\n asset=self.asset1, user=self.user, submitted=timezone.now()\n )\n\n NextReviewableTopicAsset.objects.create(\n asset=self.asset1,\n item=self.asset1.item,\n item_item_id=self.asset1.item.item_id,\n project=self.asset1.item.project,\n project_slug=self.asset1.item.project.slug,\n topic=self.topic,\n sequence=self.asset1.sequence,\n transcriber_ids=[self.user.id],\n )\n\n populate_next_reviewable_for_topic(topic_id=self.topic.id)\n mock_logger.info.assert_any_call(\n \"No reviewable assets found in topic %s\", self.topic\n )\n\n @mock.patch(\"concordia.tasks.next_asset.transcribable.logger\")\n def test_populate_next_transcribable_for_campaign_none_found(self, mock_logger):\n for asset in (self.asset1, self.asset2):\n NextTranscribableCampaignAsset.objects.create(\n asset=asset,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n campaign=self.campaign,\n sequence=asset.sequence,\n transcription_status=asset.transcription_status,\n )\n\n populate_next_transcribable_for_campaign(campaign_id=self.campaign.id)\n mock_logger.info.assert_any_call(\n \"No transcribable assets found in campaign %s\", self.campaign\n )\n\n @mock.patch(\"concordia.tasks.next_asset.transcribable.logger\")\n def test_populate_next_transcribable_for_topic_none_found(self, mock_logger):\n for asset in (self.asset1, self.asset2):\n NextTranscribableTopicAsset.objects.create(\n asset=asset,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n topic=self.topic,\n sequence=asset.sequence,\n transcription_status=asset.transcription_status,\n )\n\n populate_next_transcribable_for_topic(topic_id=self.topic.id)\n mock_logger.info.assert_any_call(\n \"No transcribable assets found in topic %s\", self.topic\n )\n\n\nclass CleanNextAssetTasksTests(TestCase):\n def setUp(self):\n self.asset = create_asset()\n self.campaign = self.asset.campaign\n self.topic = create_topic(project=self.asset.item.project)\n self.campaign_transcribable = NextTranscribableCampaignAsset.objects.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.asset.item.project,\n project_slug=self.asset.item.project.slug,\n campaign=self.campaign,\n sequence=self.asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n self.topic_transcribable = NextTranscribableTopicAsset.objects.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.asset.item.project,\n project_slug=self.asset.item.project.slug,\n topic=self.topic,\n sequence=self.asset.sequence,\n transcription_status=TranscriptionStatus.IN_PROGRESS,\n )\n self.campaign_reviewable = NextReviewableCampaignAsset.objects.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.asset.item.project,\n project_slug=self.asset.item.project.slug,\n campaign=self.campaign,\n sequence=self.asset.sequence,\n )\n self.topic_reviewable = NextReviewableTopicAsset.objects.create(\n asset=self.asset,\n item=self.asset.item,\n item_item_id=self.asset.item.item_id,\n project=self.asset.item.project,\n project_slug=self.asset.item.project.slug,\n topic=self.topic,\n sequence=self.asset.sequence,\n )\n\n @mock.patch(\n \"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_campaign.delay\"\n )\n def test_clean_next_transcribable_for_campaign(self, mock_delay):\n self.asset.transcription_status = TranscriptionStatus.COMPLETED\n self.asset.save()\n clean_next_transcribable_for_campaign(self.campaign.id)\n self.assertFalse(\n NextTranscribableCampaignAsset.objects.filter(\n campaign=self.campaign\n ).exists()\n )\n mock_delay.assert_called_once_with(self.campaign.id)\n\n @mock.patch(\n \"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_topic.delay\"\n )\n def test_clean_next_transcribable_for_topic(self, mock_delay):\n self.asset.transcription_status = TranscriptionStatus.COMPLETED\n self.asset.save()\n clean_next_transcribable_for_topic(self.topic.id)\n self.assertFalse(\n NextTranscribableTopicAsset.objects.filter(topic=self.topic).exists()\n )\n mock_delay.assert_called_once_with(self.topic.id)\n\n @mock.patch(\n \"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_campaign.delay\"\n )\n def test_clean_next_reviewable_for_campaign(self, mock_delay):\n self.asset.transcription_status = TranscriptionStatus.IN_PROGRESS\n self.asset.save()\n clean_next_reviewable_for_campaign(self.campaign.id)\n self.assertFalse(\n NextReviewableCampaignAsset.objects.filter(campaign=self.campaign).exists()\n )\n mock_delay.assert_called_once_with(self.campaign.id)\n\n @mock.patch(\n \"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_topic.delay\"\n )\n def test_clean_next_reviewable_for_topic(self, mock_delay):\n self.asset.transcription_status = TranscriptionStatus.NOT_STARTED\n self.asset.save()\n clean_next_reviewable_for_topic(self.topic.id)\n self.assertFalse(\n NextReviewableTopicAsset.objects.filter(topic=self.topic).exists()\n )\n mock_delay.assert_called_once_with(self.topic.id)\n\n @mock.patch(\n \"concordia.tasks.next_asset.reviewable.clean_next_reviewable_for_campaign.delay\"\n )\n @mock.patch(\n \"concordia.tasks.next_asset.transcribable.clean_next_transcribable_for_campaign.delay\"\n )\n @mock.patch(\n \"concordia.tasks.next_asset.reviewable.clean_next_reviewable_for_topic.delay\"\n )\n @mock.patch(\n \"concordia.tasks.next_asset.transcribable.clean_next_transcribable_for_topic.delay\"\n )\n def test_renew_next_asset_cache(\n self,\n mock_clean_trans_topic,\n mock_clean_rev_topic,\n mock_clean_trans_campaign,\n mock_clean_rev_campaign,\n ):\n renew_next_asset_cache()\n mock_clean_trans_campaign.assert_called_once_with(campaign_id=self.campaign.id)\n mock_clean_rev_campaign.assert_called_once_with(campaign_id=self.campaign.id)\n mock_clean_trans_topic.assert_called_once_with(topic_id=self.topic.id)\n mock_clean_rev_topic.assert_called_once_with(topic_id=self.topic.id)\n\n @mock.patch(\"concordia.tasks.next_asset.transcribable.logger\")\n def test_clean_next_transcribable_for_campaign_exception(self, mock_logger):\n with mock.patch.object(\n self.campaign_transcribable, \"delete\", side_effect=Exception(\"fail\")\n ):\n with mock.patch(\n \"concordia.tasks.next_asset.transcribable.find_invalid_next_transcribable_campaign_assets\",\n return_value=[self.campaign_transcribable],\n ):\n clean_next_transcribable_for_campaign(self.campaign.id)\n mock_logger.exception.assert_called_once()\n\n @mock.patch(\"concordia.tasks.next_asset.transcribable.logger\")\n def test_clean_next_transcribable_for_topic_exception(self, mock_logger):\n with mock.patch.object(\n self.topic_transcribable, \"delete\", side_effect=Exception(\"fail\")\n ):\n with mock.patch(\n \"concordia.tasks.next_asset.transcribable.find_invalid_next_transcribable_topic_assets\",\n return_value=[self.topic_transcribable],\n ):\n clean_next_transcribable_for_topic(self.topic.id)\n mock_logger.exception.assert_called_once()\n\n @mock.patch(\"concordia.tasks.next_asset.reviewable.logger\")\n def test_clean_next_reviewable_for_campaign_exception(self, mock_logger):\n with mock.patch.object(\n self.campaign_reviewable, \"delete\", side_effect=Exception(\"fail\")\n ):\n with mock.patch(\n \"concordia.tasks.next_asset.reviewable.find_invalid_next_reviewable_campaign_assets\",\n return_value=[self.campaign_reviewable],\n ):\n clean_next_reviewable_for_campaign(self.campaign.id)\n mock_logger.exception.assert_called_once()\n\n @mock.patch(\"concordia.tasks.next_asset.reviewable.logger\")\n def test_clean_next_reviewable_for_topic_exception(self, mock_logger):\n with mock.patch.object(\n self.topic_reviewable, \"delete\", side_effect=Exception(\"fail\")\n ):\n with mock.patch(\n \"concordia.tasks.next_asset.reviewable.find_invalid_next_reviewable_topic_assets\",\n return_value=[self.topic_reviewable],\n ):\n clean_next_reviewable_for_topic(self.topic.id)\n mock_logger.exception.assert_called_once()\n"
},
{
"path": "concordia/tests/test_tasks_reports_backfill.py",
"content": "from django.test import TestCase\nfrom django.utils import timezone\n\nfrom concordia.models import Campaign, SiteReport, Topic\nfrom concordia.tasks.reports.backfill import (\n backfill_assets_started_for_site_reports,\n)\n\n\nclass BackfillAssetsStartedTaskTests(TestCase):\n def _dt(self, days_ago):\n return timezone.now() - timezone.timedelta(days=days_ago)\n\n def test_updates_total_and_skips_existing_by_default(self):\n # Three TOTAL rows in time order. The last is already populated and\n # should be skipped in default mode.\n r1 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_total=100,\n assets_not_started=100,\n assets_published=10,\n assets_started=None,\n )\n r2 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_total=107,\n assets_not_started=92,\n assets_published=17,\n assets_started=None,\n )\n r3 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_total=100,\n assets_not_started=90,\n assets_published=20,\n assets_started=5,\n )\n\n dt1 = self._dt(3)\n dt2 = self._dt(2)\n dt3 = self._dt(1)\n\n SiteReport.objects.filter(pk=r1.pk).update(created_on=dt1)\n SiteReport.objects.filter(pk=r2.pk).update(created_on=dt2)\n SiteReport.objects.filter(pk=r3.pk).update(created_on=dt3)\n\n # The TOTAL series assets_started is now expected to be rolled up from\n # per-campaign series that were generated on the same reporting day.\n camp1 = Campaign.objects.create(title=\"C1\", slug=\"c1\")\n camp2 = Campaign.objects.create(title=\"C2\", slug=\"c2\")\n\n c1_prev = SiteReport.objects.create(\n campaign=camp1,\n assets_total=50,\n assets_not_started=50,\n assets_published=0,\n assets_started=None,\n )\n c1_curr = SiteReport.objects.create(\n campaign=camp1,\n assets_total=50,\n assets_not_started=40,\n assets_published=0,\n assets_started=None,\n )\n\n c2_prev = SiteReport.objects.create(\n campaign=camp2,\n assets_total=50,\n assets_not_started=50,\n assets_published=0,\n assets_started=None,\n )\n c2_curr = SiteReport.objects.create(\n campaign=camp2,\n assets_total=52,\n assets_not_started=47,\n assets_published=0,\n assets_started=None,\n )\n\n SiteReport.objects.filter(pk=c1_prev.pk).update(created_on=dt1)\n SiteReport.objects.filter(pk=c1_curr.pk).update(created_on=dt2)\n SiteReport.objects.filter(pk=c2_prev.pk).update(created_on=dt1)\n SiteReport.objects.filter(pk=c2_curr.pk).update(created_on=dt2)\n\n updated = backfill_assets_started_for_site_reports.run()\n # Campaign series (2 campaigns x 2 rows) + TOTAL series (2 rows).\n self.assertEqual(updated, 6)\n\n r1.refresh_from_db()\n r2.refresh_from_db()\n r3.refresh_from_db()\n c1_prev.refresh_from_db()\n c1_curr.refresh_from_db()\n c2_prev.refresh_from_db()\n c2_curr.refresh_from_db()\n\n self.assertEqual(c1_prev.assets_started, 0)\n self.assertEqual(c1_curr.assets_started, 10)\n self.assertEqual(c2_prev.assets_started, 0)\n self.assertEqual(c2_curr.assets_started, 5)\n\n self.assertEqual(r1.assets_started, 0)\n self.assertEqual(r2.assets_started, 15)\n self.assertEqual(r3.assets_started, 5)\n\n def test_recompute_when_skip_existing_is_false(self):\n # Build a TOTAL series with two rows. Make the first row have a wrong,\n # non-null assets_started so it should be recomputed when\n # skip_existing is False.\n #\n # The TOTAL series assets_started is expected to be rolled up from\n # per-campaign series generated on the same reporting day.\n now = timezone.now()\n\n prev = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_total=100,\n assets_not_started=100,\n assets_published=10,\n )\n curr = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_total=105,\n assets_not_started=90,\n assets_published=15,\n )\n\n dt_prev = now - timezone.timedelta(days=2)\n dt_curr = now - timezone.timedelta(days=1)\n\n # Enforce chronological order for the iterator\n SiteReport.objects.filter(pk=prev.pk).update(created_on=dt_prev)\n SiteReport.objects.filter(pk=curr.pk).update(created_on=dt_curr)\n\n # Wrong non-null on first row, null on second.\n SiteReport.objects.filter(pk=prev.pk).update(assets_started=5)\n SiteReport.objects.filter(pk=curr.pk).update(assets_started=None)\n\n camp = Campaign.objects.create(title=\"C\", slug=\"c\")\n\n c_prev = SiteReport.objects.create(\n campaign=camp,\n assets_total=100,\n assets_not_started=100,\n assets_published=0,\n assets_started=None,\n )\n c_curr = SiteReport.objects.create(\n campaign=camp,\n assets_total=105,\n assets_not_started=90,\n assets_published=0,\n assets_started=None,\n )\n SiteReport.objects.filter(pk=c_prev.pk).update(created_on=dt_prev)\n SiteReport.objects.filter(pk=c_curr.pk).update(created_on=dt_curr)\n\n updated = backfill_assets_started_for_site_reports.run(skip_existing=False)\n # Campaign series (2 rows) + TOTAL series (2 rows).\n self.assertEqual(updated, 4)\n\n prev_refreshed = SiteReport.objects.get(pk=prev.pk)\n curr_refreshed = SiteReport.objects.get(pk=curr.pk)\n c_prev.refresh_from_db()\n c_curr.refresh_from_db()\n\n self.assertEqual(c_prev.assets_started, 0)\n self.assertEqual(c_curr.assets_started, 15)\n\n self.assertEqual(prev_refreshed.assets_started, 0)\n self.assertEqual(curr_refreshed.assets_started, 15)\n\n def test_processes_retired_campaign_and_topic_series(self):\n # One RETIRED_TOTAL row\n rt = SiteReport.objects.create(\n report_name=SiteReport.ReportName.RETIRED_TOTAL,\n assets_not_started=10,\n assets_published=2,\n assets_started=None,\n )\n SiteReport.objects.filter(pk=rt.pk).update(created_on=self._dt(3))\n\n # One per-campaign row\n camp = Campaign.objects.create(title=\"C\", slug=\"c\")\n cr = SiteReport.objects.create(\n campaign=camp,\n assets_not_started=7,\n assets_published=1,\n assets_started=None,\n )\n SiteReport.objects.filter(pk=cr.pk).update(created_on=self._dt(2))\n\n # One per-topic row\n topic = Topic.objects.create(title=\"T\", slug=\"t\")\n tr = SiteReport.objects.create(\n topic=topic,\n assets_not_started=5,\n assets_published=0,\n assets_started=None,\n )\n SiteReport.objects.filter(pk=tr.pk).update(created_on=self._dt(1))\n\n updated = backfill_assets_started_for_site_reports.run()\n # Each single-row series sets assets_started to 0\n self.assertEqual(updated, 3)\n\n rt.refresh_from_db()\n cr.refresh_from_db()\n tr.refresh_from_db()\n self.assertEqual(rt.assets_started, 0)\n self.assertEqual(cr.assets_started, 0)\n self.assertEqual(tr.assets_started, 0)\n\n def test_skip_existing_branch_emits_heartbeat_due_to_time(self):\n # First row already populated (skipped); second row needs update.\n from unittest import mock\n\n prev = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_not_started=100,\n assets_published=10,\n assets_started=0,\n )\n curr = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_not_started=90,\n assets_published=15,\n assets_started=None,\n )\n\n SiteReport.objects.filter(pk=prev.pk).update(created_on=self._dt(2))\n SiteReport.objects.filter(pk=curr.pk).update(created_on=self._dt(1))\n\n # Use a monotonic function that always advances time enough to trip the\n # heartbeat-by-time condition, without exhausting side effects.\n def make_monotonic(step=11.0):\n state = {\"t\": 0.0}\n\n def _mono():\n state[\"t\"] += step\n return state[\"t\"]\n\n return _mono\n\n with (\n mock.patch(\"concordia.tasks.reports.backfill.structured_logger\") as slog,\n mock.patch(\n \"concordia.tasks.reports.backfill.time.monotonic\",\n new=make_monotonic(),\n ),\n ):\n updated = backfill_assets_started_for_site_reports.run()\n self.assertEqual(updated, 1)\n\n hb_calls = [\n c\n for c in slog.info.call_args_list\n if c.kwargs.get(\"event_code\")\n == \"assets_started_backfill_series_heartbeat\"\n and c.kwargs.get(\"series\") == \"TOTAL\"\n and c.kwargs.get(\"scanned_rows\") == 1\n and c.kwargs.get(\"last_seen_site_report_id\") == prev.id\n ]\n self.assertTrue(hb_calls)\n\n def test_post_scan_heartbeat_emitted_due_to_time(self):\n # Single-row series where a save occurs, then a heartbeat fires due to\n # elapsed time.\n from unittest import mock\n\n single = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_not_started=50,\n assets_published=5,\n assets_started=None,\n )\n SiteReport.objects.filter(pk=single.pk).update(created_on=self._dt(1))\n\n def make_monotonic(step=11.0):\n state = {\"t\": 0.0}\n\n def _mono():\n state[\"t\"] += step\n return state[\"t\"]\n\n return _mono\n\n with (\n mock.patch(\"concordia.tasks.reports.backfill.structured_logger\") as slog,\n mock.patch(\n \"concordia.tasks.reports.backfill.time.monotonic\",\n new=make_monotonic(),\n ),\n ):\n updated = backfill_assets_started_for_site_reports.run()\n self.assertEqual(updated, 1)\n\n slog.info.assert_any_call(\n \"Scanning series...\",\n event_code=\"assets_started_backfill_series_heartbeat\",\n series=\"TOTAL\",\n scanned_rows=1,\n updated_rows=1,\n last_seen_site_report_id=single.id,\n )\n\n def test_no_update_when_equal_with_skip_existing_false(self):\n # First row already equals the calculated value (zero for first snapshot),\n # so no save should occur on that row when recomputing.\n from unittest import mock\n\n prev = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_not_started=100,\n assets_published=10,\n assets_started=0,\n )\n curr = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_not_started=90,\n assets_published=15,\n assets_started=None,\n )\n\n SiteReport.objects.filter(pk=prev.pk).update(created_on=self._dt(2))\n SiteReport.objects.filter(pk=curr.pk).update(created_on=self._dt(1))\n\n with mock.patch(\"concordia.tasks.reports.backfill.structured_logger\") as slog:\n updated = backfill_assets_started_for_site_reports.run(skip_existing=False)\n self.assertEqual(updated, 1)\n\n # Row logs should not include the first row, since it already matched.\n row_logs = [\n c.kwargs\n for c in slog.info.call_args_list\n if c.kwargs.get(\"event_code\") == \"assets_started_backfill_row\"\n ]\n self.assertTrue(any(kw.get(\"site_report_id\") == curr.id for kw in row_logs))\n self.assertFalse(\n any(kw.get(\"site_report_id\") == prev.id for kw in row_logs)\n )\n\n def test_total_assets_started_is_rolled_up_from_campaign_series(self):\n # Ensure the TOTAL series does not derive assets_started from its own\n # assets_total/assets_not_started deltas when campaign series data for\n # the same reporting days exists.\n camp = Campaign.objects.create(title=\"C\", slug=\"c\")\n\n total_prev = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_total=999,\n assets_not_started=999,\n assets_started=None,\n )\n total_curr = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n assets_total=999,\n assets_not_started=0,\n assets_started=None,\n )\n\n c_prev = SiteReport.objects.create(\n campaign=camp,\n assets_total=20,\n assets_not_started=20,\n assets_started=None,\n )\n c_curr = SiteReport.objects.create(\n campaign=camp,\n assets_total=20,\n assets_not_started=15,\n assets_started=None,\n )\n\n dt_prev = self._dt(2)\n dt_curr = self._dt(1)\n\n SiteReport.objects.filter(pk=total_prev.pk).update(created_on=dt_prev)\n SiteReport.objects.filter(pk=total_curr.pk).update(created_on=dt_curr)\n SiteReport.objects.filter(pk=c_prev.pk).update(created_on=dt_prev)\n SiteReport.objects.filter(pk=c_curr.pk).update(created_on=dt_curr)\n\n updated = backfill_assets_started_for_site_reports.run(skip_existing=False)\n self.assertEqual(updated, 4)\n\n total_prev.refresh_from_db()\n total_curr.refresh_from_db()\n c_prev.refresh_from_db()\n c_curr.refresh_from_db()\n\n self.assertEqual(c_prev.assets_started, 0)\n self.assertEqual(c_curr.assets_started, 5)\n\n self.assertEqual(total_prev.assets_started, 0)\n self.assertEqual(total_curr.assets_started, 5)\n"
},
{
"path": "concordia/tests/test_tasks_reports_key_metrics.py",
"content": "from datetime import date, datetime\nfrom types import SimpleNamespace\nfrom unittest import mock\n\nfrom django.test import TestCase\nfrom django.utils import timezone\n\nfrom concordia.models import KeyMetricsReport, SiteReport\nfrom concordia.tasks.reports.key_metrics import build_key_metrics_reports\n\n\nclass BuildKeyMetricsReportsTaskTests(TestCase):\n def _dt(self, days_ago):\n return timezone.now() - timezone.timedelta(days=days_ago)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_recompute_all_calls_all_upserts(self, mock_localdate):\n # Fix \"today\" to a stable mid-month date.\n today = date(2024, 3, 15)\n mock_localdate.return_value = today\n\n # Earliest SiteReport in the current month so only one month is walked.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n tz = timezone.get_current_timezone()\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 3, 10, 12, 0, 0), tz)\n )\n\n fy = KeyMetricsReport.get_fiscal_year_for_date(today)\n fq = KeyMetricsReport.get_fiscal_quarter_for_date(today)\n\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=today.replace(day=1),\n period_end=today,\n fiscal_year=fy,\n fiscal_quarter=fq,\n month=today.month,\n )\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=today.replace(day=1),\n period_end=today,\n fiscal_year=fy,\n fiscal_quarter=fq,\n )\n\n with (\n mock.patch.object(KeyMetricsReport, \"upsert_month\") as up_m,\n mock.patch.object(KeyMetricsReport, \"upsert_quarter\") as up_q,\n mock.patch.object(KeyMetricsReport, \"upsert_fiscal_year\") as up_y,\n ):\n up_m.return_value = mock.Mock(period_start=None, period_end=None)\n up_q.return_value = mock.Mock(period_start=None, period_end=None)\n up_y.return_value = mock.Mock(period_start=None, period_end=None)\n\n changed = build_key_metrics_reports.run(recompute_all=True)\n\n # One month, four quarters, one fiscal year\n self.assertEqual(changed, 6)\n self.assertEqual(up_m.call_count, 1)\n self.assertEqual(up_q.call_count, 4)\n self.assertEqual(up_y.call_count, 1)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_incremental_refresh_and_creates(self, mock_localdate):\n # Fix \"today\" to a stable mid-month date so the month logic is\n # deterministic.\n today = date(2024, 3, 15)\n mock_localdate.return_value = today\n\n # Make one SiteReport this month so the month is considered.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n tz = timezone.get_current_timezone()\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 3, 10, 12, 0, 0), tz)\n )\n\n fy = KeyMetricsReport.get_fiscal_year_for_date(today)\n fq = KeyMetricsReport.get_fiscal_quarter_for_date(today)\n\n # Existing MONTHLY row with old updated_on so it is refreshed.\n monthly = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=today.replace(day=1),\n period_end=today,\n fiscal_year=fy,\n fiscal_quarter=fq,\n month=today.month,\n )\n KeyMetricsReport.objects.filter(pk=monthly.pk).update(\n updated_on=timezone.make_aware(datetime(2024, 3, 1, 0, 0, 0), tz)\n )\n\n # Existing QUARTERLY row for the same quarter; the other three quarters\n # are missing and will be created. We keep updated_on equal to the\n # MONTHLY row so only the missing quarters are upserted.\n quarter = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=today.replace(day=1),\n period_end=today,\n fiscal_year=fy,\n fiscal_quarter=fq,\n )\n KeyMetricsReport.objects.filter(pk=quarter.pk).update(\n updated_on=timezone.make_aware(datetime(2024, 3, 1, 0, 0, 0), tz)\n )\n\n # No FY row yet so it will be created in the FY stage.\n with (\n mock.patch.object(KeyMetricsReport, \"upsert_month\") as up_m,\n mock.patch.object(KeyMetricsReport, \"upsert_quarter\") as up_q,\n mock.patch.object(KeyMetricsReport, \"upsert_fiscal_year\") as up_y,\n ):\n up_m.return_value = mock.Mock(period_start=None, period_end=None)\n up_q.return_value = mock.Mock(period_start=None, period_end=None)\n up_y.return_value = mock.Mock(period_start=None, period_end=None)\n\n changed = build_key_metrics_reports(recompute_all=False)\n\n # One monthly refresh, three quarterly creates (no refresh since the\n # mock does not bump monthly.updated_on), and one fiscal year create.\n self.assertEqual(changed, 5)\n self.assertEqual(up_m.call_count, 1)\n self.assertEqual(up_q.call_count, 3)\n self.assertEqual(up_y.call_count, 1)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.SiteReport\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_early_return_after_backsteps(self, mock_local, mock_sr, slog):\n # Force \"today\" to mid-March so last_month_start starts at Mar 1.\n mock_local.return_value = date(2024, 3, 15)\n\n # Earliest SR is mid-December so first_month_start is Dec 1.\n earliest = SimpleNamespace(\n created_on=timezone.make_aware(datetime(2023, 12, 15, 12, 0, 0))\n )\n mock_sr.objects.order_by.return_value.first.return_value = earliest\n\n # Pretend there are no snapshots by EOM for any month we check.\n mock_sr.objects.filter.return_value.exists.return_value = False\n\n changed = build_key_metrics_reports(recompute_all=False)\n self.assertEqual(changed, 0)\n\n # Ensure we logged the \"no months\" message.\n codes = [kw.get(\"event_code\") for _, kw in slog.info.call_args_list if kw]\n self.assertIn(\"key_metrics_build_no_months\", codes)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_recompute_all_month_upsert_and_december_rollover(\n self, mock_local, upsert_month, slog\n ):\n # Make yesterday in December so the month we process is December.\n mock_local.return_value = date(2023, 12, 20)\n\n # Create a TOTAL snapshot in December so the scan does not early-return.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2023, 12, 15, 10, 0, 0))\n )\n\n # Return a stub report so the \"upserted\" logging runs.\n upsert_month.return_value = SimpleNamespace(\n period_start=date(2023, 12, 1),\n period_end=date(2023, 12, 31),\n )\n\n changed = build_key_metrics_reports(recompute_all=True)\n self.assertGreaterEqual(changed, 1)\n\n codes = [kw.get(\"event_code\") for _, kw in slog.info.call_args_list if kw]\n self.assertIn(\"key_metrics_month_upserted\", codes)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_incremental_month_create_and_refresh(\n self,\n mock_local,\n upsert_month,\n upsert_year,\n upsert_quarter,\n slog,\n ):\n mock_local.return_value = date(2024, 2, 1)\n\n sr_jan = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr_jan.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))\n )\n sr_dec = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr_dec.pk).update(\n created_on=timezone.make_aware(datetime(2023, 12, 20, 9, 0, 0))\n )\n\n dec_month = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2023, 12, 1),\n period_end=date(2023, 12, 31),\n fiscal_year=2024,\n fiscal_quarter=1,\n month=12,\n )\n KeyMetricsReport.objects.filter(pk=dec_month.pk).update(\n updated_on=timezone.make_aware(datetime(2023, 12, 1, 0, 0, 0))\n )\n\n # Monthly upsert produces a stub (so it counts as 1 change per call)\n upsert_month.return_value = SimpleNamespace(\n period_start=date(2024, 1, 1), period_end=date(2024, 1, 31)\n )\n # Disable quarterly and fiscal-year increments\n upsert_quarter.return_value = None\n upsert_year.return_value = None\n\n changed = build_key_metrics_reports(recompute_all=False)\n self.assertEqual(changed, 2)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.objects\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_quarter_recompute_all_logs(\n self, mock_local, upsert_month, upsert_quarter, kmr_objects, slog\n ):\n mock_local.return_value = date(2024, 1, 15)\n\n # Ensure we do not early-return (one SR anywhere is fine).\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 1, 8, 0, 0))\n )\n\n # We are not using monthly upserts here.\n upsert_month.return_value = None\n\n # monthly_rows -> one fiscal year (2024)\n kmr_objects.filter.return_value.values.return_value.annotate.return_value = [\n {\"fiscal_year\": 2024}\n ]\n # quarter_exists .first() can be anything; ignored in recompute_all.\n kmr_objects.filter.return_value.first.return_value = None\n # Prevent FY stage from running by returning no quarter years later.\n kmr_objects.filter.return_value.values_list.return_value = []\n\n upsert_quarter.return_value = SimpleNamespace(\n period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)\n )\n\n changed = build_key_metrics_reports(recompute_all=True)\n # Four quarters upserted\n self.assertGreaterEqual(changed, 4)\n self.assertEqual(upsert_quarter.call_count, 4)\n\n codes = [kw.get(\"event_code\") for _, kw in slog.info.call_args_list if kw]\n self.assertIn(\"key_metrics_quarter_upserted\", codes)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.objects\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_quarter_incremental_refresh_all_quarters(\n self, mock_local, upsert_month, kmr_objects, upsert_quarter, slog\n ):\n mock_local.return_value = date(2024, 6, 15)\n\n # Ensure we do not early-return.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 5, 10, 8, 0, 0))\n )\n\n # No monthly creation in this test.\n upsert_month.return_value = None\n\n # Signal that we have monthly rows for fiscal_year=2024.\n kmr_objects.filter.return_value.values.return_value.annotate.return_value = [\n {\"fiscal_year\": 2024}\n ]\n\n # quarter_exists present for all four quarters.\n quarter_stub = SimpleNamespace(\n updated_on=timezone.make_aware(datetime(2024, 1, 1, 0, 0, 0))\n )\n\n def filter_side_effect(*args, **kwargs):\n # For QUARTERLY lookups with fiscal_quarter, return an object\n # whose first() yields a stub so \"refresh\" path is taken.\n class QS:\n def __init__(self, exists_value=False):\n self._exists = exists_value\n\n def first(self):\n return quarter_stub\n\n def exists(self):\n return self._exists\n\n def values(self, *a, **k):\n return self\n\n def annotate(self, *a, **k):\n return [{\"fiscal_year\": 2024}]\n\n def values_list(self, *a, **k):\n # Avoid FY stage in this test\n return []\n\n pt = kwargs.get(\"period_type\")\n if pt == KeyMetricsReport.PeriodType.MONTHLY and \"updated_on__gt\" in kwargs:\n # Make monthly_newer_exists True\n return QS(exists_value=True)\n return QS()\n\n kmr_objects.filter.side_effect = filter_side_effect\n\n upsert_quarter.return_value = SimpleNamespace(\n period_start=date(2024, 4, 1), period_end=date(2024, 6, 30)\n )\n\n changed = build_key_metrics_reports(recompute_all=False)\n # Four refreshes (Q1..Q4)\n self.assertGreaterEqual(changed, 4)\n self.assertEqual(upsert_quarter.call_count, 4)\n\n codes = [kw.get(\"event_code\") for _, kw in slog.info.call_args_list if kw]\n self.assertIn(\"key_metrics_quarter_refreshed\", codes)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.objects\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_fiscal_year_recompute_all_logs(\n self, mock_local, upsert_month, kmr_objects, upsert_year, slog\n ):\n mock_local.return_value = date(2024, 1, 15)\n\n # Ensure no early-return.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 2, 8, 0, 0))\n )\n\n upsert_month.return_value = None\n\n # No monthlies needed; quarters present for FY 2027.\n kmr_objects.filter.return_value.values.return_value.annotate.return_value = []\n kmr_objects.filter.return_value.values_list.return_value = [2027]\n kmr_objects.filter.return_value.first.return_value = None\n\n upsert_year.return_value = SimpleNamespace(\n period_start=date(2026, 10, 1), period_end=date(2027, 9, 30)\n )\n\n changed = build_key_metrics_reports(recompute_all=True)\n self.assertGreaterEqual(changed, 1)\n\n codes = [kw.get(\"event_code\") for _, kw in slog.info.call_args_list if kw]\n self.assertIn(\"key_metrics_year_upserted\", codes)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.objects\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_fiscal_year_incremental_create_and_refresh(\n self, mock_local, upsert_month, kmr_objects, upsert_year, slog\n ):\n mock_local.return_value = date(2024, 5, 1)\n\n # Ensure no early-return.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 4, 15, 8, 0, 0))\n )\n upsert_month.return_value = None\n\n # First, drive \"create\" path: quarters exist, FY row is missing.\n def filter_values_list_side_effect(*args, **kwargs):\n # This handles the \"fiscal_years_with_quarters\" query.\n class QS:\n def values_list(self, *a, **k):\n return [2026]\n\n def first(self):\n return None\n\n def values(self, *a, **k):\n return self\n\n def annotate(self, *a, **k):\n return []\n\n def exists(self):\n return False\n\n return QS()\n\n kmr_objects.filter.side_effect = filter_values_list_side_effect\n\n upsert_year.return_value = SimpleNamespace(\n period_start=date(2025, 10, 1), period_end=date(2026, 9, 30)\n )\n\n changed1 = build_key_metrics_reports(recompute_all=False)\n self.assertGreaterEqual(changed1, 1)\n codes1 = [kw.get(\"event_code\") for _, kw in slog.info.call_args_list if kw]\n self.assertIn(\"key_metrics_year_created\", codes1)\n\n # Now drive \"refresh\" path: FY exists, a newer quarter exists.\n fy_stub = SimpleNamespace(\n updated_on=timezone.make_aware(datetime(2024, 3, 1, 0, 0, 0))\n )\n\n def filter_refresh_side_effect(*args, **kwargs):\n class QS:\n def __init__(self, pt=None):\n self.pt = pt\n\n def values_list(self, *a, **k):\n return [2026]\n\n def first(self):\n # When asking for the FY row, return a stub\n return fy_stub\n\n def values(self, *a, **k):\n return self\n\n def annotate(self, *a, **k):\n return []\n\n def exists(self):\n # This is called for quarters newer than FY.updated_on\n return True\n\n return QS()\n\n kmr_objects.filter.side_effect = filter_refresh_side_effect\n\n upsert_year.return_value = SimpleNamespace(\n period_start=date(2025, 10, 1), period_end=date(2026, 9, 30)\n )\n\n changed2 = build_key_metrics_reports(recompute_all=False)\n self.assertGreaterEqual(changed2, 1)\n codes2 = [kw.get(\"event_code\") for _, kw in slog.info.call_args_list if kw]\n self.assertIn(\"key_metrics_year_refreshed\", codes2)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_recompute_all_quarter_upserts_only(\n self, mock_local, mock_month, mock_quarter, mock_year, slog\n ):\n mock_local.return_value = date(2024, 2, 1)\n\n # Seed one site snapshot so the task has a start month (Jan 2024).\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))\n )\n\n # Seed a MONTHLY row so the quarter loop sees FY 2024 in the set.\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 1, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n month=1,\n )\n\n # Monthly does nothing; quarter upserts return a stub; FY does nothing.\n mock_month.return_value = None\n mock_year.return_value = None\n\n def quarter_stub(**kwargs):\n return SimpleNamespace(\n period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)\n )\n\n mock_quarter.side_effect = quarter_stub\n\n changed = build_key_metrics_reports(recompute_all=True)\n\n # Only quarters (4) should have counted.\n self.assertEqual(changed, 4)\n self.assertEqual(mock_quarter.call_count, 4)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_incremental_quarter_refresh_only(\n self, mock_local, mock_month, mock_quarter, mock_year, slog\n ):\n mock_local.return_value = date(2024, 4, 1)\n\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))\n )\n\n # Monthlies for Q2; newer than the quarter row we will refresh\n for m in (1, 2, 3):\n mr = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, m, 1),\n period_end=KeyMetricsReport.month_bounds(date(2024, m, 15))[1],\n fiscal_year=2024,\n fiscal_quarter=2,\n month=m,\n )\n KeyMetricsReport.objects.filter(pk=mr.pk).update(\n updated_on=timezone.make_aware(datetime(2024, 3, 31, 12, 0, 0))\n )\n\n # Pre-create Q1, Q3, Q4 so they are not created by the task\n for fq, ps, pe in [\n (1, date(2023, 10, 1), date(2023, 12, 31)),\n (3, date(2024, 4, 1), date(2024, 6, 30)),\n (4, date(2024, 7, 1), date(2024, 9, 30)),\n ]:\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=ps,\n period_end=pe,\n fiscal_year=2024,\n fiscal_quarter=fq,\n )\n\n # Existing Q2 with older updated_on so only this quarter refreshes\n q2 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n KeyMetricsReport.objects.filter(pk=q2.pk).update(\n updated_on=timezone.make_aware(datetime(2024, 1, 15, 0, 0, 0))\n )\n\n mock_month.return_value = None\n mock_year.return_value = None\n mock_quarter.return_value = SimpleNamespace(\n period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)\n )\n\n changed = build_key_metrics_reports(recompute_all=False)\n\n self.assertEqual(changed, 1)\n self.assertEqual(mock_quarter.call_count, 1)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_recompute_all_year_upsert_only(\n self, mock_local, mock_month, mock_quarter, mock_year, slog\n ):\n mock_local.return_value = date(2024, 2, 1)\n\n # Seed snapshot to allow the task to pick a month.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))\n )\n\n # Ensure the 'fiscal_years_with_quarters' set is not empty.\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n\n # Monthly and quarterly stages do nothing; FY upsert returns a stub.\n mock_month.return_value = None\n mock_quarter.return_value = None\n mock_year.return_value = SimpleNamespace(\n period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)\n )\n\n changed = build_key_metrics_reports(recompute_all=True)\n\n self.assertEqual(changed, 1)\n self.assertEqual(mock_year.call_count, 1)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_incremental_year_create(\n self, mock_local, mock_month, mock_quarter, mock_year, slog\n ):\n mock_local.return_value = date(2024, 2, 1)\n\n # Seed snapshot and a quarterly row so year loop triggers.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))\n )\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n\n mock_month.return_value = None\n mock_quarter.return_value = None\n mock_year.return_value = SimpleNamespace(\n period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)\n )\n\n changed = build_key_metrics_reports(recompute_all=False)\n\n self.assertEqual(changed, 1)\n self.assertEqual(mock_year.call_count, 1)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_incremental_year_refresh(\n self, mock_local, mock_month, mock_quarter, mock_year, slog\n ):\n mock_local.return_value = date(2024, 4, 1)\n\n # Seed a quarterly row with new updated_on.\n q = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n KeyMetricsReport.objects.filter(pk=q.pk).update(\n updated_on=timezone.make_aware(datetime(2024, 3, 31, 12, 0, 0))\n )\n\n # Create an older FY row that should be refreshed.\n fy = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,\n period_start=date(2023, 10, 1),\n period_end=date(2024, 9, 30),\n fiscal_year=2024,\n )\n KeyMetricsReport.objects.filter(pk=fy.pk).update(\n updated_on=timezone.make_aware(datetime(2024, 1, 1, 0, 0, 0))\n )\n\n # Need a snapshot so the task can initialize months; it is not used\n # further because we neutralize month and quarter stages.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))\n )\n\n mock_month.return_value = None\n mock_quarter.return_value = None\n mock_year.return_value = SimpleNamespace(\n period_start=date(2023, 10, 1), period_end=date(2024, 9, 30)\n )\n\n changed = build_key_metrics_reports(recompute_all=False)\n\n self.assertEqual(changed, 1)\n self.assertEqual(mock_year.call_count, 1)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_quarter_recompute_all_upserts_and_continue(\n self,\n mock_localdate,\n mock_upsert_month,\n mock_upsert_quarter,\n mock_upsert_year,\n slog,\n ):\n # Make the \"monthly\" section inert (no changes).\n mock_localdate.return_value = date(2024, 4, 1)\n mock_upsert_month.return_value = None\n mock_upsert_year.return_value = None\n\n # Seed minimal SiteReport so the monthly stage can compute bounds safely.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))\n )\n\n # Ensure at least one fiscal_year is discovered from MONTHLY rows.\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 1, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n month=1,\n )\n\n # Each quarter upsert returns a non-None object so rows_changed increments.\n mock_upsert_quarter.return_value = SimpleNamespace(\n period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)\n )\n\n changed = build_key_metrics_reports.run(recompute_all=True)\n\n # Four quarters upserted; monthly and FY upserts return None.\n self.assertEqual(changed, 4)\n self.assertEqual(mock_upsert_quarter.call_count, 4)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_fiscal_year_recompute_all_upserts_and_continue(\n self,\n mock_localdate,\n mock_upsert_month,\n mock_upsert_quarter,\n mock_upsert_year,\n slog,\n ):\n mock_localdate.return_value = date(2024, 4, 1)\n mock_upsert_month.return_value = None\n mock_upsert_quarter.return_value = None\n\n # Seed a quarter so the FY stage finds a fiscal year to process.\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n\n # Earliest SiteReport so earlier stages do not error.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 5, 9, 0, 0))\n )\n\n mock_upsert_year.return_value = SimpleNamespace(\n period_start=date(2023, 10, 1), period_end=date(2024, 9, 30)\n )\n\n changed = build_key_metrics_reports.run(recompute_all=True)\n\n # Only FY upsert counts (quarter/month upserts return None).\n self.assertEqual(changed, 1)\n self.assertEqual(mock_upsert_year.call_count, 1)\n\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\",\n return_value=None,\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_quarter_recompute_all_non_none_continue_edge(\n self, mock_localdate, mock_upsert_quarter, mock_upsert_month\n ):\n mock_localdate.return_value = date(2024, 5, 20)\n\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0))\n )\n\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 5, 1),\n period_end=date(2024, 5, 31),\n fiscal_year=2024,\n fiscal_quarter=3,\n month=5,\n )\n\n dummy = mock.MagicMock(\n period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)\n )\n mock_upsert_quarter.return_value = dummy\n\n changed = build_key_metrics_reports(recompute_all=True)\n\n self.assertEqual(changed, 4)\n self.assertEqual(mock_upsert_quarter.call_count, 4)\n\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\",\n return_value=mock.MagicMock(\n period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)\n ),\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_quarter_incremental_refresh_monthly_newer(\n self,\n mock_localdate,\n mock_upsert_fy,\n mock_upsert_quarter,\n mock_upsert_month,\n ):\n mock_localdate.return_value = date(2024, 1, 20)\n\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))\n )\n\n jan = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 1, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n month=1,\n )\n KeyMetricsReport.objects.filter(pk=jan.pk).update(updated_on=timezone.now())\n\n now = timezone.now()\n older = now - timezone.timedelta(days=10)\n q1 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2023, 10, 1),\n period_end=date(2023, 12, 31),\n fiscal_year=2024,\n fiscal_quarter=1,\n )\n q2 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n q3 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 4, 1),\n period_end=date(2024, 6, 30),\n fiscal_year=2024,\n fiscal_quarter=3,\n )\n q4 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 7, 1),\n period_end=date(2024, 9, 30),\n fiscal_year=2024,\n fiscal_quarter=4,\n )\n KeyMetricsReport.objects.filter(pk=q1.pk).update(updated_on=now)\n KeyMetricsReport.objects.filter(pk=q2.pk).update(updated_on=older)\n KeyMetricsReport.objects.filter(pk=q3.pk).update(updated_on=now)\n KeyMetricsReport.objects.filter(pk=q4.pk).update(updated_on=now)\n\n fy = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,\n period_start=date(2023, 10, 1),\n period_end=date(2024, 9, 30),\n fiscal_year=2024,\n )\n KeyMetricsReport.objects.filter(pk=fy.pk).update(updated_on=now)\n\n mock_upsert_quarter.return_value = mock.MagicMock(\n period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)\n )\n\n changed = build_key_metrics_reports(recompute_all=False)\n\n self.assertEqual(changed, 1)\n self.assertGreaterEqual(mock_upsert_quarter.call_count, 1)\n\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\",\n return_value=mock.MagicMock(\n period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)\n ),\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_fiscal_year_recompute_all_non_none_continue_edge(\n self, mock_localdate, mock_upsert_fy, mock_upsert_quarter, mock_upsert_month\n ):\n mock_localdate.return_value = date(2024, 5, 20)\n\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0))\n )\n\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n\n changed = build_key_metrics_reports(recompute_all=True)\n self.assertEqual(changed, 1)\n\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\",\n return_value=mock.MagicMock(\n period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)\n ),\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_fiscal_year_incremental_create_missing(\n self,\n mock_localdate,\n mock_upsert_fy,\n mock_upsert_quarter,\n mock_upsert_month,\n ):\n mock_localdate.return_value = date(2024, 5, 20)\n\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n tz = timezone.get_current_timezone()\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0), tz)\n )\n\n for qn, start, end in [\n (1, date(2023, 10, 1), date(2023, 12, 31)),\n (2, date(2024, 1, 1), date(2024, 3, 31)),\n (3, date(2024, 4, 1), date(2024, 6, 30)),\n (4, date(2024, 7, 1), date(2024, 9, 30)),\n ]:\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=start,\n period_end=end,\n fiscal_year=2024,\n fiscal_quarter=qn,\n )\n\n changed = build_key_metrics_reports(recompute_all=False)\n self.assertEqual(changed, 1)\n\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\",\n return_value=mock.MagicMock(\n period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)\n ),\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_fiscal_year_incremental_refresh_when_quarter_newer(\n self, mock_localdate, mock_upsert_fy, mock_upsert_quarter, mock_upsert_month\n ):\n mock_localdate.return_value = date(2024, 5, 20)\n\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0))\n )\n\n older = timezone.now() - timezone.timedelta(days=7)\n newer = timezone.now()\n\n fy = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,\n period_start=date(2023, 10, 1),\n period_end=date(2024, 9, 30),\n fiscal_year=2024,\n )\n KeyMetricsReport.objects.filter(pk=fy.pk).update(updated_on=older)\n\n q2 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n KeyMetricsReport.objects.filter(pk=q2.pk).update(updated_on=newer)\n\n changed = build_key_metrics_reports(recompute_all=False)\n self.assertEqual(changed, 1)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\",\n return_value=None,\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_quarter_recompute_all_none_branch_continue(\n self, mock_localdate, upsert_quarter, upsert_month, upsert_year, slog\n ):\n # Keep the monthly scan minimal and stable\n mock_localdate.return_value = date(2024, 2, 10)\n\n # Seed a site snapshot so the task computes month bounds\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(\n datetime(2024, 2, 9, 12, 0, 0), timezone.get_current_timezone()\n )\n )\n\n # Ensure the quarterly stage iterates a fiscal year by having a MONTHLY\n # row\n fy = KeyMetricsReport.get_fiscal_year_for_date(mock_localdate.return_value)\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 2, 1),\n period_end=date(2024, 2, 29),\n fiscal_year=fy,\n fiscal_quarter=2,\n month=2,\n )\n\n # upsert_quarter returns None -> branch falls through to 'continue'\n changed = build_key_metrics_reports(recompute_all=True)\n\n # No rows changed because monthly and FY are neutralized and quarter\n # upserts return None (hitting the continue path each time).\n self.assertEqual(changed, 0)\n self.assertEqual(upsert_quarter.call_count, 4)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\",\n return_value=None,\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_quarter_incremental_refresh_none_branch_continue(\n self,\n mock_localdate,\n mock_upsert_quarter,\n mock_upsert_month,\n mock_upsert_year,\n slog,\n ):\n # Ensure monthly scan has a valid window\n mock_localdate.return_value = date(2024, 2, 10)\n\n # Seed one site snapshot so month range can be computed\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n tz = timezone.get_current_timezone()\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 5, 12, 0, 0), tz)\n )\n\n # Provide a MONTHLY row in FY 2024; make it \"newer\" than Q2\n jan = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.MONTHLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 1, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n month=1,\n )\n KeyMetricsReport.objects.filter(pk=jan.pk).update(updated_on=timezone.now())\n\n # Create quarter rows so the incremental branch runs.\n # Only Q2 should be older than the monthly row to trigger refresh.\n now = timezone.now()\n older = now - timezone.timedelta(days=10)\n quarters = {\n 1: ((date(2023, 10, 1), date(2023, 12, 31)), now),\n 2: ((date(2024, 1, 1), date(2024, 3, 31)), older),\n 3: ((date(2024, 4, 1), date(2024, 6, 30)), now),\n 4: ((date(2024, 7, 1), date(2024, 9, 30)), now),\n }\n for fq, val in quarters.items():\n (ps, pe), updated = val\n q = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=ps,\n period_end=pe,\n fiscal_year=2024,\n fiscal_quarter=fq,\n )\n KeyMetricsReport.objects.filter(pk=q.pk).update(updated_on=updated)\n\n # upsert_quarter is mocked to return None, so when the code reaches the\n # monthly_newer_exists refresh path for Q2 it will take the \"is None\"\n # branch and continue without incrementing rows_changed.\n changed = build_key_metrics_reports(recompute_all=False)\n\n # No rows changed: month and year upserts return None, and Q2 refresh\n # returned None (so branch continued). Only one refresh attempt expected.\n self.assertEqual(changed, 0)\n self.assertEqual(mock_upsert_quarter.call_count, 1)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\",\n return_value=None,\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_fiscal_year_recompute_all_none_branch_continue(\n self,\n mock_localdate,\n mock_upsert_month,\n mock_upsert_quarter,\n mock_upsert_year,\n slog,\n ):\n mock_localdate.return_value = date(2024, 5, 20)\n\n # Ensure monthly scan can initialize.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n tz = timezone.get_current_timezone()\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0), tz)\n )\n\n # Ensure at least one fiscal year is present for the FY stage.\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n\n # Month/quarter upserts are mocked to None; FY upsert also None.\n changed = build_key_metrics_reports(recompute_all=True)\n\n # Nothing should be counted since FY upsert returned None and the code\n # immediately continued the loop without incrementing or logging.\n self.assertEqual(changed, 0)\n self.assertEqual(mock_upsert_year.call_count, 1)\n\n codes = [kw.get(\"event_code\") for _, kw in slog.info.call_args_list if kw]\n self.assertNotIn(\"key_metrics_year_upserted\", codes)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\",\n return_value=None,\n )\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\",\n return_value=None,\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_fiscal_year_incremental_refresh_none_branch_continue(\n self,\n mock_localdate,\n mock_upsert_month,\n mock_upsert_quarter,\n mock_upsert_year,\n slog,\n ):\n mock_localdate.return_value = date(2024, 5, 20)\n\n # Make monthly stage computable.\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n tz = timezone.get_current_timezone()\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0), tz)\n )\n\n # Existing FY row with older updated_on so a newer quarter will\n # trigger the refresh path.\n fy = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,\n period_start=date(2023, 10, 1),\n period_end=date(2024, 9, 30),\n fiscal_year=2024,\n )\n KeyMetricsReport.objects.filter(pk=fy.pk).update(\n updated_on=timezone.make_aware(datetime(2024, 3, 1, 0, 0, 0), tz)\n )\n\n # Quarter newer than the FY row to make quarter_newer_exists True.\n q2 = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n KeyMetricsReport.objects.filter(pk=q2.pk).update(\n updated_on=timezone.make_aware(datetime(2024, 3, 15, 0, 0, 0), tz)\n )\n\n # FY upsert returns None so the branch is skipped and loop continues.\n changed = build_key_metrics_reports(recompute_all=False)\n\n self.assertEqual(changed, 0)\n self.assertEqual(mock_upsert_year.call_count, 1)\n\n codes = [kw.get(\"event_code\") for _, kw in slog.info.call_args_list if kw]\n self.assertNotIn(\"key_metrics_year_refreshed\", codes)\n self.assertNotIn(\"key_metrics_year_created\", codes)\n self.assertNotIn(\"key_metrics_year_upserted\", codes)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_incremental_fiscal_year_created_branch(\n self,\n mock_localdate,\n mock_upsert_month,\n mock_upsert_quarter,\n mock_upsert_year,\n slog,\n ):\n mock_localdate.return_value = date(2024, 4, 1)\n mock_upsert_month.return_value = None\n mock_upsert_quarter.return_value = None\n\n # Quarter exists for FY discovery; no FY row exists yet.\n KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 2, 9, 0, 0))\n )\n\n mock_upsert_year.return_value = SimpleNamespace(\n period_start=date(2023, 10, 1), period_end=date(2024, 9, 30)\n )\n\n changed = build_key_metrics_reports(recompute_all=False)\n\n self.assertEqual(changed, 1)\n self.assertEqual(mock_upsert_year.call_count, 1)\n\n @mock.patch(\"concordia.tasks.reports.key_metrics.structured_logger\")\n @mock.patch(\n \"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year\"\n )\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month\")\n @mock.patch(\"concordia.tasks.reports.key_metrics.timezone.localdate\")\n def test_incremental_fiscal_year_refresh_due_to_newer_quarter(\n self,\n mock_localdate,\n mock_upsert_month,\n mock_upsert_quarter,\n mock_upsert_year,\n slog,\n ):\n mock_localdate.return_value = date(2024, 4, 1)\n mock_upsert_month.return_value = None\n mock_upsert_quarter.return_value = None\n\n # Existing FY row with earlier updated_on.\n fy = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,\n period_start=date(2023, 10, 1),\n period_end=date(2024, 9, 30),\n fiscal_year=2024,\n )\n KeyMetricsReport.objects.filter(pk=fy.pk).update(\n updated_on=timezone.make_aware(datetime(2024, 3, 1, 0, 0, 0))\n )\n\n # Quarter with newer updated_on to trigger the refresh path.\n q = KeyMetricsReport.objects.create(\n period_type=KeyMetricsReport.PeriodType.QUARTERLY,\n period_start=date(2024, 1, 1),\n period_end=date(2024, 3, 31),\n fiscal_year=2024,\n fiscal_quarter=2,\n )\n KeyMetricsReport.objects.filter(pk=q.pk).update(\n updated_on=timezone.make_aware(datetime(2024, 3, 15, 0, 0, 0))\n )\n\n sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)\n SiteReport.objects.filter(pk=sr.pk).update(\n created_on=timezone.make_aware(datetime(2024, 1, 3, 9, 0, 0))\n )\n\n mock_upsert_year.return_value = SimpleNamespace(\n period_start=date(2023, 10, 1), period_end=date(2024, 9, 30)\n )\n\n changed = build_key_metrics_reports(recompute_all=False)\n\n self.assertEqual(changed, 1)\n self.assertEqual(mock_upsert_year.call_count, 1)\n"
},
{
"path": "concordia/tests/test_tasks_reports_sitereport.py",
"content": "from datetime import timedelta\n\nfrom django.test import TestCase\nfrom django.utils import timezone\n\nfrom concordia.models import Asset, Campaign, SiteReport, Transcription\nfrom concordia.tasks.reports.sitereport import (\n _daily_active_users,\n campaign_report,\n retired_total_report,\n site_report,\n)\nfrom concordia.utils import get_anonymous_user\n\nfrom .utils import (\n CreateTestUsers,\n create_asset,\n create_campaign,\n create_item,\n create_project,\n create_tag,\n create_tag_collection,\n create_topic,\n create_transcription,\n)\n\n\nclass SiteReportTestCase(CreateTestUsers, TestCase):\n @classmethod\n def setUpTestData(cls):\n # We use setUpTestData instead of setUp so the database is only set\n # up once rather than for each individual test in this test case\n cls.user1 = cls.create_user(username=\"tester1\")\n cls.user2 = cls.create_user(username=\"tester2\")\n cls.user3 = cls.create_user(username=\"tester3\")\n cls.anonymous_user = get_anonymous_user()\n cls.asset1 = create_asset()\n cls.item1 = cls.asset1.item\n cls.project1 = cls.item1.project\n cls.campaign1 = cls.project1.campaign\n cls.asset1_transcription1 = create_transcription(\n asset=cls.asset1, user=cls.user1, accepted=timezone.now()\n )\n cls.asset1_transcription2 = create_transcription(\n asset=cls.asset1,\n user=cls.anonymous_user,\n rejected=timezone.now(),\n reviewed_by=cls.user1,\n )\n cls.topic1 = create_topic(project=cls.project1)\n cls.tag1 = create_tag()\n cls.tag_collection1 = create_tag_collection(\n tag=cls.tag1, asset=cls.asset1, user=cls.user1\n )\n\n cls.campaign2 = create_campaign(slug=\"test-campaign-slug-2\")\n cls.project2 = create_project(\n campaign=cls.campaign2, slug=\"test-project-slug-2\"\n )\n cls.item2 = create_item(project=cls.project2, item_id=\"2\")\n cls.asset2 = create_asset(item=cls.item2, slug=\"test-asset-slug-2\")\n cls.topic2 = create_topic(\n project=cls.asset2.item.project, slug=\"test-topic-slug-2\"\n )\n cls.tag_collection2 = create_tag_collection(\n tag=cls.tag1, asset=cls.asset2, user=cls.user1\n )\n\n cls.campaign3 = create_campaign(slug=\"test-campaign-slug-3\")\n cls.project3 = create_project(\n campaign=cls.campaign3, slug=\"test-project-slug-3\"\n )\n cls.item3 = create_item(project=cls.project3, item_id=\"3\")\n cls.asset3 = create_asset(item=cls.item3, slug=\"test-asset-slug-3\")\n cls.asset4 = create_asset(\n item=cls.item3, slug=\"test-asset-slug-4\", published=False\n )\n cls.item4 = create_item(project=cls.project3, item_id=\"4\", published=False)\n cls.asset5 = create_asset(\n item=cls.item4, slug=\"test-asset-slug-5\", published=False\n )\n\n cls.project3.topics.add(cls.topic1)\n cls.project3.topics.add(cls.topic2)\n\n cls.retired_campaign = create_campaign(slug=\"retired-campaign-slug\")\n cls.retired_project = create_project(\n campaign=cls.retired_campaign, slug=\"retired-project-slug\"\n )\n cls.retired_item = create_item(project=cls.retired_project)\n cls.retired_asset = create_asset(\n item=cls.retired_item, slug=\"retired-asset-slug\"\n )\n time = timezone.now() - timedelta(days=1, hours=1)\n cls.retired_asset_transcription1 = create_transcription(\n asset=cls.retired_asset, user=cls.user1, accepted=time\n )\n # Done like this to override auto_now_add and auto_now\n Transcription.objects.filter(pk=cls.retired_asset_transcription1.pk).update(\n created_on=time, updated_on=time\n )\n time = timezone.now() - timedelta(days=1, seconds=1)\n cls.retired_asset_transcription2 = create_transcription(\n asset=cls.retired_asset,\n user=cls.user2,\n rejected=time,\n reviewed_by=cls.user1,\n )\n # Done like this to override auto_now_add and auto_now\n Transcription.objects.filter(pk=cls.retired_asset_transcription2.pk).update(\n created_on=time, updated_on=time\n )\n\n # Generate the campaign report before \"retiring\" the campaign to populate\n # the retired total report\n cls.retired_campaign_report = campaign_report(campaign=cls.retired_campaign)\n cls.retired_asset.delete()\n cls.retired_item.delete()\n cls.retired_project.delete()\n cls.retired_campaign.status = Campaign.Status.RETIRED\n cls.retired_campaign.save()\n\n site_report()\n cls.site_report = SiteReport.objects.filter(\n report_name=SiteReport.ReportName.TOTAL\n ).first()\n cls.retired_site_report = SiteReport.objects.filter(\n report_name=SiteReport.ReportName.RETIRED_TOTAL\n ).first()\n cls.campaign1_report = SiteReport.objects.filter(campaign=cls.campaign1).first()\n cls.topic1_report = SiteReport.objects.filter(topic=cls.topic1).first()\n\n def test_daily_active_users(self):\n self.assertEqual(_daily_active_users(), 2)\n\n def test_site_report(self):\n self.assertEqual(self.site_report.assets_total, 5)\n self.assertEqual(self.site_report.assets_published, 3)\n self.assertEqual(self.site_report.assets_not_started, 4)\n self.assertEqual(self.site_report.assets_in_progress, 1)\n self.assertEqual(self.site_report.assets_waiting_review, 0)\n self.assertEqual(self.site_report.assets_completed, 0)\n self.assertEqual(self.site_report.assets_unpublished, 2)\n self.assertEqual(self.site_report.items_published, 3)\n self.assertEqual(self.site_report.items_unpublished, 1)\n self.assertEqual(self.site_report.projects_published, 3)\n self.assertEqual(self.site_report.projects_unpublished, 0)\n self.assertEqual(self.site_report.anonymous_transcriptions, 1)\n self.assertEqual(self.site_report.transcriptions_saved, 2)\n self.assertEqual(self.site_report.daily_review_actions, 2)\n self.assertEqual(self.site_report.distinct_tags, 1)\n self.assertEqual(self.site_report.tag_uses, 2)\n self.assertEqual(self.site_report.campaigns_published, 4)\n self.assertEqual(self.site_report.campaigns_unpublished, 0)\n self.assertEqual(self.site_report.users_registered, 4)\n self.assertEqual(self.site_report.users_activated, 4)\n self.assertEqual(self.site_report.daily_active_users, 2)\n\n def test_retired_site_report(self):\n self.assertEqual(self.retired_site_report.assets_total, 1)\n self.assertEqual(self.retired_site_report.assets_published, 1)\n self.assertEqual(self.retired_site_report.assets_not_started, 0)\n self.assertEqual(self.retired_site_report.assets_in_progress, 1)\n self.assertEqual(self.retired_site_report.assets_waiting_review, 0)\n self.assertEqual(self.retired_site_report.assets_completed, 0)\n self.assertEqual(self.retired_site_report.assets_unpublished, 0)\n self.assertEqual(self.retired_site_report.items_published, 1)\n self.assertEqual(self.retired_site_report.items_unpublished, 0)\n self.assertEqual(self.retired_site_report.projects_published, 1)\n self.assertEqual(self.retired_site_report.projects_unpublished, 0)\n self.assertEqual(self.retired_site_report.anonymous_transcriptions, 0)\n self.assertEqual(self.retired_site_report.transcriptions_saved, 2)\n self.assertEqual(self.retired_site_report.daily_review_actions, 0)\n self.assertEqual(self.retired_site_report.distinct_tags, 0)\n self.assertEqual(self.retired_site_report.tag_uses, 0)\n self.assertEqual(self.retired_site_report.registered_contributors, 2)\n\n def test_campaign_report(self):\n self.assertEqual(self.campaign1_report.assets_total, 1)\n self.assertEqual(self.campaign1_report.assets_published, 1)\n self.assertEqual(self.campaign1_report.assets_not_started, 0)\n self.assertEqual(self.campaign1_report.assets_in_progress, 1)\n self.assertEqual(self.campaign1_report.assets_waiting_review, 0)\n self.assertEqual(self.campaign1_report.assets_completed, 0)\n self.assertEqual(self.campaign1_report.assets_unpublished, 0)\n self.assertEqual(self.campaign1_report.items_published, 1)\n self.assertEqual(self.campaign1_report.items_unpublished, 0)\n self.assertEqual(self.campaign1_report.projects_published, 1)\n self.assertEqual(self.campaign1_report.projects_unpublished, 0)\n self.assertEqual(self.campaign1_report.anonymous_transcriptions, 1)\n self.assertEqual(self.campaign1_report.transcriptions_saved, 2)\n self.assertEqual(self.campaign1_report.daily_review_actions, 2)\n self.assertEqual(self.campaign1_report.distinct_tags, 1)\n self.assertEqual(self.campaign1_report.tag_uses, 1)\n self.assertEqual(self.campaign1_report.registered_contributors, 2)\n\n def test_topic_report(self):\n self.assertEqual(self.topic1_report.assets_total, 4)\n self.assertEqual(self.topic1_report.assets_published, 2)\n self.assertEqual(self.topic1_report.assets_not_started, 3)\n self.assertEqual(self.topic1_report.assets_in_progress, 1)\n self.assertEqual(self.topic1_report.assets_waiting_review, 0)\n self.assertEqual(self.topic1_report.assets_completed, 0)\n self.assertEqual(self.topic1_report.assets_unpublished, 2)\n self.assertEqual(self.topic1_report.items_published, 2)\n self.assertEqual(self.topic1_report.items_unpublished, 1)\n self.assertEqual(self.topic1_report.projects_published, 2)\n self.assertEqual(self.topic1_report.projects_unpublished, 0)\n self.assertEqual(self.topic1_report.anonymous_transcriptions, 1)\n self.assertEqual(self.topic1_report.transcriptions_saved, 2)\n self.assertEqual(self.topic1_report.daily_review_actions, 2)\n self.assertEqual(self.topic1_report.distinct_tags, 1)\n self.assertEqual(self.topic1_report.tag_uses, 1)\n\n def test_topic_report_zero_assets_emits_warning(self):\n # Create a new topic attached to a project with no items/assets so the\n # topic report computes zero total assets and emits a warning.\n from unittest import mock\n\n empty_campaign = create_campaign(slug=\"sr-empty-c\")\n empty_project = create_project(campaign=empty_campaign, slug=\"sr-empty-p\")\n empty_topic = create_topic(project=empty_project, slug=\"sr-empty-t\")\n\n with mock.patch(\"concordia.tasks.reports.sitereport.structured_logger\") as slog:\n site_report()\n\n warn_calls = [\n c\n for c in slog.warning.call_args_list\n if c.kwargs.get(\"event_code\") == \"topic_report_zero_assets\"\n and c.kwargs.get(\"topic\") == empty_topic\n ]\n self.assertTrue(warn_calls)\n\n\nclass SiteReportAssetsStartedRollupTests(CreateTestUsers, TestCase):\n def test_total_assets_started_rolls_up_campaign_deltas_ignoring_retirements(\n self,\n ):\n \"\"\"\n The TOTAL assets_started value should be derived from per-campaign\n daily deltas for the same reporting day.\n\n This protects the site-wide daily count from being suppressed when a\n campaign retires and its already-started assets are removed from the\n active asset tables.\n \"\"\"\n from unittest import mock\n\n active_campaign = create_campaign(slug=\"rollup-active-c\")\n retiring_campaign = create_campaign(slug=\"rollup-retiring-c\")\n\n active_project = create_project(\n campaign=active_campaign, slug=\"rollup-active-p\"\n )\n active_item = create_item(project=active_project, item_id=\"ra\")\n active_asset = create_asset(item=active_item, slug=\"rollup-active-a\")\n\n retiring_project = create_project(\n campaign=retiring_campaign, slug=\"rollup-retiring-p\"\n )\n retiring_item = create_item(project=retiring_project, item_id=\"rb\")\n retiring_asset = create_asset(item=retiring_item, slug=\"rollup-retiring-a\")\n\n # Day 1 snapshot: one not-started asset in the active campaign and one\n # already-started asset in the campaign that will retire.\n Asset.objects.filter(pk=active_asset.pk).update(\n transcription_status=\"not_started\"\n )\n Asset.objects.filter(pk=retiring_asset.pk).update(\n transcription_status=\"in_progress\"\n )\n\n base_now = timezone.now()\n day1 = base_now - timedelta(days=2)\n day2 = base_now - timedelta(days=1)\n\n with mock.patch(\"django.utils.timezone.now\", return_value=day1):\n site_report()\n\n # Between snapshots, the active campaign starts its asset.\n Asset.objects.filter(pk=active_asset.pk).update(\n transcription_status=\"in_progress\"\n )\n\n # The other campaign is retired and its content is removed.\n retiring_asset.delete()\n retiring_item.delete()\n retiring_project.delete()\n retiring_campaign.status = Campaign.Status.RETIRED\n retiring_campaign.save()\n\n with mock.patch(\"django.utils.timezone.now\", return_value=day2):\n site_report()\n\n total_day2 = (\n SiteReport.objects.filter(\n report_name=SiteReport.ReportName.TOTAL,\n campaign__isnull=True,\n topic__isnull=True,\n created_on__date=day2.date(),\n )\n .order_by(\"-created_on\", \"-pk\")\n .first()\n )\n active_day2 = (\n SiteReport.objects.filter(\n campaign=active_campaign, created_on__date=day2.date()\n )\n .order_by(\"-created_on\", \"-pk\")\n .first()\n )\n\n self.assertIsNotNone(total_day2)\n self.assertIsNotNone(active_day2)\n\n self.assertEqual(active_day2.assets_started, 1)\n self.assertEqual(total_day2.assets_started, 1)\n\n def test_retired_total_assets_started_is_always_zero(self):\n retired_campaign = create_campaign(slug=\"rollup-retired-c\")\n retired_campaign.status = Campaign.Status.RETIRED\n retired_campaign.save()\n\n r1 = SiteReport.objects.create(\n campaign=retired_campaign,\n assets_total=10,\n assets_not_started=0,\n assets_started=4,\n )\n r2 = SiteReport.objects.create(\n campaign=retired_campaign,\n assets_total=10,\n assets_not_started=0,\n assets_started=7,\n )\n\n now = timezone.now()\n SiteReport.objects.filter(pk=r1.pk).update(created_on=now - timedelta(days=2))\n SiteReport.objects.filter(pk=r2.pk).update(created_on=now - timedelta(days=1))\n\n self.assertEqual(SiteReport.objects.get(pk=r2.pk).assets_started, 7)\n\n retired_total_report()\n\n retired_total = (\n SiteReport.objects.filter(report_name=SiteReport.ReportName.RETIRED_TOTAL)\n .order_by(\"-created_on\", \"-pk\")\n .first()\n )\n\n self.assertIsNotNone(retired_total)\n self.assertEqual(retired_total.assets_started, 0)\n"
},
{
"path": "concordia/tests/test_tasks_retirement.py",
"content": "from unittest import mock\n\nfrom django.core.exceptions import ObjectDoesNotExist\nfrom django.test import TestCase\n\nfrom concordia.models import Asset, Project\nfrom concordia.tasks.retirement import (\n assets_removal_success,\n delete_asset,\n item_removal_success,\n project_removal_success,\n remove_next_assets,\n remove_next_item,\n remove_next_project,\n retire_campaign,\n)\n\nfrom .utils import (\n create_asset,\n create_campaign,\n create_campaign_retirement_progress,\n create_item,\n create_project,\n)\n\n\nclass RetirementTasksTests(TestCase):\n def test_retire_campaign_initializes_totals_and_sets_status_and_triggers(self):\n # Build a campaign with 2 projects, 2 items, 3 assets.\n camp = create_campaign(slug=\"ret-c1\")\n p1 = create_project(campaign=camp, slug=\"ret-p1\")\n p2 = create_project(campaign=camp, slug=\"ret-p2\")\n i1 = create_item(project=p1, item_id=\"ret-i1\")\n i2 = create_item(project=p2, item_id=\"ret-i2\")\n a1 = create_asset(item=i1, slug=\"ret-a1\")\n a2 = create_asset(item=i1, slug=\"ret-a2\")\n a3 = create_asset(item=i2, slug=\"ret-a3\")\n self.assertTrue(all([a1.pk, a2.pk, a3.pk]))\n\n with mock.patch(\n \"concordia.tasks.retirement.remove_next_project.delay\"\n ) as m_delay:\n prog = retire_campaign(camp.id)\n\n prog.refresh_from_db()\n self.assertEqual(prog.project_total, 2)\n self.assertEqual(prog.item_total, 2)\n self.assertEqual(prog.asset_total, 3)\n camp.refresh_from_db()\n # Status must be set to RETIRED.\n self.assertEqual(camp.status, camp.Status.RETIRED) # type: ignore[attr-defined]\n m_delay.assert_called_once_with(camp.id)\n\n def test_retire_campaign_existing_progress_and_already_retired(self):\n camp = create_campaign(slug=\"ret-c2\")\n # Pre-create progress so the totals branch is skipped.\n prog = create_campaign_retirement_progress(campaign=camp)\n prog.project_total = 7\n prog.item_total = 8\n prog.asset_total = 9\n prog.save()\n # Mark campaign retired to skip status change.\n camp.status = camp.Status.RETIRED # type: ignore[attr-defined]\n camp.save()\n\n with mock.patch(\n \"concordia.tasks.retirement.remove_next_project.delay\"\n ) as m_delay:\n retire_campaign(camp.id)\n\n prog.refresh_from_db()\n self.assertEqual(prog.project_total, 7)\n self.assertEqual(prog.item_total, 8)\n self.assertEqual(prog.asset_total, 9)\n camp.refresh_from_db()\n self.assertEqual(camp.status, camp.Status.RETIRED) # type: ignore[attr-defined]\n m_delay.assert_called_once_with(camp.id)\n\n def test_remove_next_project_calls_remove_next_item_when_project_exists(self):\n camp = create_campaign(slug=\"ret-c3\")\n proj = create_project(campaign=camp, slug=\"ret-p3\")\n create_campaign_retirement_progress(campaign=camp)\n\n with mock.patch(\"concordia.tasks.retirement.remove_next_item.delay\") as m_delay:\n remove_next_project(camp.id)\n\n m_delay.assert_called_once_with(proj.id)\n\n def test_remove_next_project_marks_complete_when_no_projects(self):\n camp = create_campaign(slug=\"ret-c4\")\n prog = create_campaign_retirement_progress(campaign=camp)\n\n with mock.patch(\"concordia.tasks.retirement.remove_next_item.delay\") as m_delay:\n remove_next_project(camp.id)\n\n prog.refresh_from_db()\n self.assertTrue(prog.complete)\n self.assertIsNotNone(prog.completed_on)\n m_delay.assert_not_called()\n\n def test_project_removal_success_increments_and_triggers_next(self):\n camp = create_campaign(slug=\"ret-c5\")\n prog = create_campaign_retirement_progress(campaign=camp)\n self.assertEqual(prog.projects_removed, 0)\n\n with mock.patch(\n \"concordia.tasks.retirement.remove_next_project.delay\"\n ) as m_delay:\n project_removal_success(project_id=123, campaign_id=camp.id)\n\n prog.refresh_from_db()\n self.assertEqual(prog.projects_removed, 1)\n self.assertTrue(any(e.get(\"id\") == 123 for e in prog.removal_log))\n m_delay.assert_called_once_with(camp.id)\n\n def test_remove_next_item_calls_remove_next_assets_when_item_exists(self):\n camp = create_campaign(slug=\"ret-c6\")\n proj = create_project(campaign=camp, slug=\"ret-p6\")\n itm = create_item(project=proj, item_id=\"ret-i6\")\n\n with mock.patch(\n \"concordia.tasks.retirement.remove_next_assets.delay\"\n ) as m_delay:\n remove_next_item(proj.id)\n\n m_delay.assert_called_once_with(itm.id)\n\n def test_remove_next_item_deletes_project_and_triggers_when_no_items(self):\n camp = create_campaign(slug=\"ret-c7\")\n proj = create_project(campaign=camp, slug=\"ret-p7\")\n\n with mock.patch(\n \"concordia.tasks.retirement.project_removal_success.delay\"\n ) as m_delay:\n remove_next_item(proj.id)\n\n with self.assertRaises(ObjectDoesNotExist):\n Project.objects.get(pk=proj.id)\n m_delay.assert_called_once_with(proj.id, camp.id)\n\n def test_assets_removal_success_updates_counts_and_triggers_next(self):\n camp = create_campaign(slug=\"ret-c8\")\n prog = create_campaign_retirement_progress(campaign=camp)\n self.assertEqual(prog.assets_removed, 0)\n\n with mock.patch(\n \"concordia.tasks.retirement.remove_next_assets.delay\"\n ) as m_delay:\n assets_removal_success([10, 11, 12], campaign_id=camp.id, item_id=55)\n\n prog.refresh_from_db()\n self.assertEqual(prog.assets_removed, 3)\n self.assertTrue(any(e.get(\"id\") == 10 for e in prog.removal_log))\n self.assertTrue(any(e.get(\"id\") == 11 for e in prog.removal_log))\n self.assertTrue(any(e.get(\"id\") == 12 for e in prog.removal_log))\n m_delay.assert_called_once_with(55)\n\n def test_remove_next_assets_when_no_assets_deletes_item_and_triggers(self):\n camp = create_campaign(slug=\"ret-c9\")\n proj = create_project(campaign=camp, slug=\"ret-p9\")\n itm = create_item(project=proj, item_id=\"ret-i9\")\n\n with mock.patch(\n \"concordia.tasks.retirement.item_removal_success.delay\"\n ) as m_delay:\n remove_next_assets(itm.id)\n\n with self.assertRaises(ObjectDoesNotExist):\n # Item should be deleted.\n type(itm).objects.get(pk=itm.id) # type: ignore[attr-defined]\n m_delay.assert_called_once_with(itm.id, camp.id, proj.id)\n\n def test_remove_next_assets_with_assets_uses_chord_in_chunks_of_10(self):\n camp = create_campaign(slug=\"ret-c10\")\n proj = create_project(campaign=camp, slug=\"ret-p10\")\n itm = create_item(project=proj, item_id=\"ret-i10\")\n # Create 12 assets; only 10 should be in the chord header.\n ids = []\n for n in range(12):\n a = create_asset(item=itm, slug=f\"ret-a10-{n}\", sequence=n)\n ids.append(a.id)\n first_ten = list(\n Asset.objects.filter(item=itm)\n .order_by(\"id\")\n .values_list(\"id\", flat=True)[:10]\n )\n\n with (\n mock.patch(\"concordia.tasks.retirement.chord\") as m_chord,\n mock.patch(\"concordia.tasks.retirement.delete_asset.s\") as m_del_sig,\n mock.patch(\n \"concordia.tasks.retirement.assets_removal_success.s\"\n ) as m_body_sig,\n ):\n runner = mock.MagicMock()\n m_chord.return_value = runner\n m_del_sig.side_effect = lambda aid: f\"S({aid})\"\n m_body_sig.return_value = \"BODY\"\n\n remove_next_assets(itm.id)\n\n # Header should contain exactly 10 signatures, matching first ten ids.\n header_iter = m_chord.call_args[0][0]\n header_list = list(header_iter)\n self.assertEqual(header_list, [f\"S({aid})\" for aid in first_ten])\n # The body signature should be called with campaign and item ids.\n m_body_sig.assert_called_once_with(camp.id, itm.id)\n runner.assert_called_once_with(\"BODY\")\n\n def test_delete_asset_deletes_storage_and_model_and_returns_id(self):\n itm = create_item(item_id=\"ret-i11\")\n a = create_asset(item=itm, slug=\"ret-a11\", sequence=11)\n\n with mock.patch(\"django.core.files.storage.FileSystemStorage.delete\") as m_del:\n ret_id = delete_asset(a.id)\n\n self.assertEqual(ret_id, a.id)\n self.assertFalse(Asset.objects.filter(pk=a.id).exists())\n m_del.assert_called()\n\n def test_item_removal_success_increments_and_triggers_next(self):\n camp = create_campaign(slug=\"ret-c12\")\n proj = create_project(campaign=camp, slug=\"ret-p12\")\n itm = create_item(project=proj, item_id=\"ret-i12\")\n prog = create_campaign_retirement_progress(campaign=camp)\n self.assertEqual(prog.items_removed, 0)\n\n with mock.patch(\"concordia.tasks.retirement.remove_next_item.delay\") as m_delay:\n item_removal_success(\n item_id=itm.id, campaign_id=camp.id, project_id=proj.id\n )\n\n prog.refresh_from_db()\n self.assertEqual(prog.items_removed, 1)\n self.assertTrue(\n any(\n entry.get(\"type\") == \"item\" and entry.get(\"id\") == itm.id\n for entry in prog.removal_log\n )\n )\n m_delay.assert_called_once_with(proj.id)\n"
},
{
"path": "concordia/tests/test_tasks_search_index.py",
"content": "from unittest import mock\n\nfrom django.test import TestCase\n\nfrom concordia.tasks.search_index import (\n create_opensearch_indices,\n delete_opensearch_indices,\n populate_opensearch_assets_indices,\n populate_opensearch_indices,\n populate_opensearch_users_indices,\n rebuild_opensearch_indices,\n)\n\n\nclass SearchIndexTasksTests(TestCase):\n def test_create_opensearch_indices_calls_management_command(self):\n with mock.patch(\"concordia.tasks.search_index.call_command\") as m_call:\n result = create_opensearch_indices()\n self.assertIsNone(result)\n m_call.assert_called_once_with(\n \"opensearch\",\n \"index\",\n \"create\",\n verbosity=2,\n force=True,\n ignore_error=True,\n )\n\n def test_delete_opensearch_indices_calls_management_command(self):\n with mock.patch(\"concordia.tasks.search_index.call_command\") as m_call:\n result = delete_opensearch_indices()\n self.assertIsNone(result)\n m_call.assert_called_once_with(\n \"opensearch\", \"index\", \"delete\", force=True, ignore_error=True\n )\n\n def test_rebuild_opensearch_indices_calls_management_command(self):\n with mock.patch(\"concordia.tasks.search_index.call_command\") as m_call:\n result = rebuild_opensearch_indices()\n self.assertIsNone(result)\n m_call.assert_called_once_with(\n \"opensearch\",\n \"index\",\n \"rebuild\",\n verbosity=2,\n force=True,\n ignore_error=True,\n )\n\n def test_populate_users_indices_calls_management_command(self):\n with mock.patch(\"concordia.tasks.search_index.call_command\") as m_call:\n result = populate_opensearch_users_indices()\n self.assertIsNone(result)\n m_call.assert_called_once_with(\n \"opensearch\",\n \"document\",\n \"index\",\n \"--indices\",\n \"users\",\n \"--force\",\n \"--parallel\",\n )\n\n def test_populate_assets_indices_calls_management_command(self):\n with mock.patch(\"concordia.tasks.search_index.call_command\") as m_call:\n result = populate_opensearch_assets_indices()\n self.assertIsNone(result)\n m_call.assert_called_once_with(\n \"opensearch\",\n \"document\",\n \"index\",\n \"--indices\",\n \"assets\",\n \"--force\",\n \"--parallel\",\n )\n\n def test_populate_all_indices_calls_management_command(self):\n with mock.patch(\"concordia.tasks.search_index.call_command\") as m_call:\n result = populate_opensearch_indices()\n self.assertIsNone(result)\n m_call.assert_called_once_with(\n \"opensearch\", \"document\", \"index\", \"--force\", \"--parallel\"\n )\n"
},
{
"path": "concordia/tests/test_tasks_thumbnails.py",
"content": "from unittest import mock\n\nfrom django.test import TestCase\n\nfrom concordia.tasks.thumbnails import (\n download_item_thumbnail_task,\n download_missing_thumbnails_task,\n)\n\nfrom .utils import create_campaign, create_item, create_project\n\n\nclass ThumbnailsTasksTests(TestCase):\n def test_download_item_thumbnail_task_returns_skip_when_no_url(self):\n # Item has no thumbnail_url; task should return skip message.\n proj = create_project(campaign=create_campaign(slug=\"t-c1\"), slug=\"t-p1\")\n item = create_item(project=proj, item_id=\"t-i1\", thumbnail_url=\"\")\n\n with mock.patch(\"importer.tasks.items.download_and_set_item_thumbnail\") as m_dl:\n result = download_item_thumbnail_task.run(item.id, force=False)\n\n self.assertEqual(result, \"No thumbnail URL available.\")\n m_dl.assert_not_called()\n\n def test_download_item_thumbnail_task_calls_helper_with_force(self):\n # Item has a thumbnail_url; helper should be called with force flag.\n proj = create_project(campaign=create_campaign(slug=\"t-c2\"), slug=\"t-p2\")\n item = create_item(\n project=proj,\n item_id=\"t-i2\",\n thumbnail_url=\"https://ex.invalid/t.jpg\",\n thumbnail_image=\"\",\n )\n\n with mock.patch(\"importer.tasks.items.download_and_set_item_thumbnail\") as m_dl:\n m_dl.return_value = \"stored/path/t.jpg\"\n result = download_item_thumbnail_task.run(item.id, force=True)\n\n self.assertEqual(result, \"stored/path/t.jpg\")\n m_dl.assert_called_once()\n # First arg is the Item instance, second the source URL.\n args, kwargs = m_dl.call_args\n self.assertEqual(args[0].id, item.id)\n self.assertEqual(args[1], \"https://ex.invalid/t.jpg\")\n self.assertTrue(kwargs.get(\"force\"))\n\n def test_download_missing_thumbnails_task_returns_zero_when_none(self):\n # No items meet the filter; should log and return 0, no group calls.\n with mock.patch(\"concordia.tasks.thumbnails.group\") as m_group:\n count = download_missing_thumbnails_task.run()\n\n self.assertEqual(count, 0)\n m_group.assert_not_called()\n\n def test_download_missing_thumbnails_task_filters_and_batches_once(self):\n from unittest import mock\n\n camp = create_campaign(slug=\"t-c3\")\n proj_a = create_project(campaign=camp, slug=\"t-p3a\")\n proj_b = create_project(campaign=camp, slug=\"t-p3b\")\n\n i1 = create_item(\n project=proj_a,\n item_id=\"t-i3-1\",\n thumbnail_url=\"http://example.com/img1.jpg\",\n thumbnail_image=\"\",\n )\n i2 = create_item(\n project=proj_a,\n item_id=\"t-i3-2\",\n thumbnail_url=\"http://example.com/img2.jpg\",\n thumbnail_image=\"\",\n )\n create_item( # wrong project -> not eligible\n project=proj_b,\n item_id=\"t-i3-3\",\n thumbnail_url=\"http://example.com/img3.jpg\",\n thumbnail_image=\"\",\n )\n create_item( # already has image -> not eligible\n project=proj_a,\n item_id=\"t-i3-4\",\n thumbnail_url=\"http://example.com/img4.jpg\",\n thumbnail_image=\"has-file\",\n )\n\n with (\n mock.patch(\"concordia.tasks.thumbnails.group\") as m_group,\n mock.patch(\n \"concordia.tasks.thumbnails.download_item_thumbnail_task.s\"\n ) as m_sig,\n ):\n runner = mock.MagicMock()\n runner.apply_async.return_value.get.return_value = None\n\n def fake_group(header_iter):\n # Force generator evaluation so .s(...) is actually called.\n list(header_iter)\n return runner\n\n m_group.side_effect = fake_group\n\n count = download_missing_thumbnails_task.run(\n project_id=proj_a.id, batch_size=2, limit=10, force=True\n )\n\n self.assertEqual(count, 2)\n m_sig.assert_has_calls(\n [mock.call(i1.id, force=True), mock.call(i2.id, force=True)],\n any_order=False,\n )\n m_group.assert_called_once()\n runner.apply_async.assert_called_once()\n runner.apply_async.return_value.get.assert_called_once_with(\n disable_sync_subtasks=False\n )\n\n def test_download_missing_thumbnails_task_multiple_waves(self):\n from unittest import mock\n\n camp = create_campaign(slug=\"t-c4\")\n proj = create_project(campaign=camp, slug=\"t-p4\")\n items = [\n create_item(\n project=proj,\n item_id=f\"t-i4-{n}\",\n thumbnail_url=f\"http://example.com/{n}.jpg\",\n thumbnail_image=\"\",\n )\n for n in range(5)\n ]\n\n with (\n mock.patch(\"concordia.tasks.thumbnails.group\") as m_group,\n mock.patch(\n \"concordia.tasks.thumbnails.download_item_thumbnail_task.s\"\n ) as m_sig,\n ):\n runners = [mock.MagicMock() for _ in range(3)]\n for r in runners:\n r.apply_async.return_value.get.return_value = None\n it = iter(runners)\n\n def fake_group(header_iter):\n # Force generator consumption each wave.\n list(header_iter)\n return next(it)\n\n m_group.side_effect = fake_group\n\n count = download_missing_thumbnails_task.run(\n project_id=proj.id, batch_size=2, limit=None, force=False\n )\n\n self.assertEqual(count, 5)\n self.assertEqual(m_group.call_count, 3)\n expected = [mock.call(itm.id, force=False) for itm in items]\n self.assertEqual(m_sig.call_args_list, expected)\n"
},
{
"path": "concordia/tests/test_tasks_unusualactivity.py",
"content": "from datetime import UTC, datetime, timedelta\nfrom unittest import mock\n\nfrom django.test import TestCase, override_settings\nfrom django.utils import timezone\n\nfrom concordia.tasks.unusualactivity import unusual_activity\n\n\nclass UnusualActivityTaskTests(TestCase):\n @override_settings(CONCORDIA_ENVIRONMENT=\"development\")\n def test_noop_when_not_production_and_not_ignored(self):\n # Should not render templates or send mail.\n with (\n mock.patch(\n \"concordia.tasks.unusualactivity.loader.get_template\"\n ) as m_get_tmpl,\n mock.patch(\n \"concordia.tasks.unusualactivity.EmailMultiAlternatives\"\n ) as m_email,\n mock.patch(\n \"concordia.tasks.unusualactivity.Transcription.objects\"\n ) as m_mgr,\n ):\n unusual_activity(ignore_env=False)\n\n m_get_tmpl.assert_not_called()\n m_email.assert_not_called()\n # Manager methods should not be called either.\n self.assertFalse(m_mgr.transcribe_incidents.called)\n self.assertFalse(m_mgr.review_incidents.called)\n\n @override_settings(\n CONCORDIA_ENVIRONMENT=\"production\",\n DEFAULT_FROM_EMAIL=\"noreply@example.com\",\n DEFAULT_TO_EMAIL=\"\",\n )\n def test_runs_in_production_without_default_to(self):\n # Executes, builds subject without env suffix, and sends to one addr.\n fixed_now_dt = timezone.make_aware(datetime(2025, 1, 1, 12, 0), timezone=UTC)\n expected_one_day_ago = fixed_now_dt - timedelta(days=1)\n\n with (\n mock.patch(\n \"concordia.tasks.unusualactivity.Site.objects.get_current\"\n ) as m_site,\n mock.patch(\n \"concordia.tasks.unusualactivity.timezone.localtime\"\n ) as m_localtime,\n mock.patch(\n \"concordia.tasks.unusualactivity.timezone.now\",\n return_value=fixed_now_dt,\n ),\n mock.patch(\n \"concordia.tasks.unusualactivity.loader.get_template\"\n ) as m_get_tmpl,\n mock.patch(\n \"concordia.tasks.unusualactivity.EmailMultiAlternatives\"\n ) as m_email,\n mock.patch(\n \"concordia.tasks.unusualactivity.Transcription.objects\"\n ) as m_mgr,\n ):\n lt = mock.Mock()\n lt.strftime.return_value = \"STAMP\"\n m_localtime.return_value = lt\n\n m_site.return_value = mock.Mock(domain=\"example.com\")\n\n txt_tmpl = mock.Mock()\n html_tmpl = mock.Mock()\n txt_tmpl.render.return_value = \"TEXT\"\n html_tmpl.render.return_value = \"HTML\"\n m_get_tmpl.side_effect = lambda name: (\n txt_tmpl if name.endswith(\".txt\") else html_tmpl\n )\n\n m_mgr.transcribe_incidents.return_value = []\n m_mgr.review_incidents.return_value = []\n\n msg = mock.Mock()\n m_email.return_value = msg\n\n unusual_activity(ignore_env=False)\n\n expected_subject = \"Unusual User Activity Report for STAMP\"\n args, kwargs = m_email.call_args\n self.assertEqual(kwargs[\"subject\"], expected_subject)\n self.assertEqual(kwargs[\"from_email\"], \"noreply@example.com\")\n self.assertEqual(kwargs[\"to\"], [\"rsar@loc.gov\"])\n self.assertEqual(kwargs[\"reply_to\"], [\"noreply@example.com\"])\n\n txt_tmpl.render.assert_called_once()\n html_tmpl.render.assert_called_once()\n self.assertEqual(\n m_mgr.transcribe_incidents.call_args[0][0], expected_one_day_ago\n )\n self.assertEqual(m_mgr.review_incidents.call_args[0][0], expected_one_day_ago)\n msg.attach_alternative.assert_called_once_with(\"HTML\", \"text/html\")\n msg.send.assert_called_once()\n\n @override_settings(\n CONCORDIA_ENVIRONMENT=\"test\",\n DEFAULT_FROM_EMAIL=\"notify@example.com\",\n DEFAULT_TO_EMAIL=\"extra@example.com\",\n )\n def test_ignore_env_appends_suffix_and_includes_default_to(self):\n fixed_now_dt = timezone.make_aware(datetime(2025, 1, 2, 9, 30), timezone=UTC)\n expected_one_day_ago = fixed_now_dt - timedelta(days=1)\n\n with (\n mock.patch(\n \"concordia.tasks.unusualactivity.Site.objects.get_current\"\n ) as m_site,\n mock.patch(\n \"concordia.tasks.unusualactivity.timezone.localtime\"\n ) as m_localtime,\n mock.patch(\n \"concordia.tasks.unusualactivity.timezone.now\",\n return_value=fixed_now_dt,\n ),\n mock.patch(\n \"concordia.tasks.unusualactivity.loader.get_template\"\n ) as m_get_tmpl,\n mock.patch(\n \"concordia.tasks.unusualactivity.EmailMultiAlternatives\"\n ) as m_email,\n mock.patch(\n \"concordia.tasks.unusualactivity.Transcription.objects\"\n ) as m_mgr,\n ):\n lt = mock.Mock()\n lt.strftime.return_value = \"STAMP2\"\n m_localtime.return_value = lt\n\n m_site.return_value = mock.Mock(domain=\"example.net\")\n\n txt_tmpl = mock.Mock()\n html_tmpl = mock.Mock()\n txt_tmpl.render.return_value = \"TEXT2\"\n html_tmpl.render.return_value = \"HTML2\"\n m_get_tmpl.side_effect = lambda name: (\n txt_tmpl if name.endswith(\".txt\") else html_tmpl\n )\n\n m_mgr.transcribe_incidents.return_value = []\n m_mgr.review_incidents.return_value = []\n\n msg = mock.Mock()\n m_email.return_value = msg\n\n unusual_activity(ignore_env=True)\n\n expected_subject = \"Unusual User Activity Report for STAMP2 [TEST]\"\n args, kwargs = m_email.call_args\n self.assertEqual(kwargs[\"subject\"], expected_subject)\n self.assertEqual(kwargs[\"to\"], [\"rsar@loc.gov\", \"extra@example.com\"])\n self.assertEqual(kwargs[\"from_email\"], \"notify@example.com\")\n self.assertEqual(kwargs[\"reply_to\"], [\"notify@example.com\"])\n\n txt_tmpl.render.assert_called_once()\n html_tmpl.render.assert_called_once()\n self.assertEqual(\n m_mgr.transcribe_incidents.call_args[0][0], expected_one_day_ago\n )\n self.assertEqual(m_mgr.review_incidents.call_args[0][0], expected_one_day_ago)\n msg.attach_alternative.assert_called_once_with(\"HTML2\", \"text/html\")\n msg.send.assert_called_once()\n"
},
{
"path": "concordia/tests/test_tasks_useractivity.py",
"content": "from unittest import mock\n\nfrom django.core import mail\nfrom django.core.cache import cache\nfrom django.test import TestCase\n\nfrom concordia.exceptions import CacheLockedError\nfrom concordia.models import Campaign, Transcription, UserProfileActivity\nfrom concordia.tasks.unusualactivity import unusual_activity\nfrom concordia.tasks.useractivity import (\n populate_active_campaign_counts,\n populate_completed_campaign_counts,\n update_useractivity_cache,\n update_userprofileactivity_from_cache,\n)\nfrom concordia.utils import get_anonymous_user\n\nfrom .utils import (\n CreateTestUsers,\n create_asset,\n create_campaign,\n create_item,\n create_project,\n create_tag,\n create_tag_collection,\n create_transcription,\n)\n\n\nclass UserActivityTaskTestCase(CreateTestUsers, TestCase):\n def setUp(self):\n cache.clear()\n self.user = self.create_test_user()\n self.campaign = create_campaign()\n self.key = f\"userprofileactivity_{self.campaign.pk}\"\n\n @mock.patch(\"concordia.tasks.useractivity.update_userprofileactivity_table\")\n def test_update_userprofileactivity_from_cache_no_updates(self, mock_update_table):\n cache.set(self.key, None)\n with mock.patch(\"concordia.logging.ConcordiaLogger.debug\") as mock_debug:\n update_userprofileactivity_from_cache()\n self.assertEqual(mock_debug.call_count, 2)\n mock_debug.assert_called_with(\n \"Cache contained no updates for key. Skipping\",\n event_code=\"update_userprofileactivity_from_cache_no_updates\",\n key=self.key,\n )\n self.assertEqual(mock_update_table.call_count, 0)\n\n @mock.patch(\"concordia.tasks.useractivity.update_userprofileactivity_table\")\n def test_update_userprofileactivity_from_cache_update(self, mock_update_table):\n cache.set(self.key, {self.user.pk: (1, 0)})\n update_userprofileactivity_from_cache()\n self.assertEqual(mock_update_table.call_count, 2)\n mock_update_table.assert_has_calls(\n [\n mock.call(self.user, self.campaign.id, \"transcribe_count\", 1),\n mock.call(self.user, self.campaign.id, \"review_count\", 0),\n ]\n )\n self.assertIsNone(cache.get(self.key))\n\n @mock.patch(\"concordia.tasks.unusualactivity.Transcription.objects\")\n def test_unusual_activity(self, mock_transcription):\n mock_transcription.transcribe_incidents.return_value = (\n Transcription.objects.none()\n )\n mock_transcription.review_incidents.return_value = Transcription.objects.none()\n unusual_activity(ignore_env=True)\n self.assertEqual(len(mail.outbox), 1)\n expected_subject = \"Unusual User Activity Report\"\n self.assertIn(expected_subject, mail.outbox[0].subject)\n\n @mock.patch(\"django.core.cache.cache.add\")\n @mock.patch(\"django.core.cache.cache.delete\")\n @mock.patch(\"concordia.tasks.useractivity._update_useractivity_cache\")\n def test_update_useractivity_cache(self, mock_update, mock_delete, mock_add):\n user = self.user\n campaign = self.campaign\n\n mock_add.return_value = False\n with self.assertRaises(CacheLockedError):\n update_useractivity_cache(user.id, campaign.id, \"transcribe\")\n self.assertEqual(mock_update.call_count, 0)\n self.assertEqual(mock_delete.call_count, 0)\n\n mock_add.return_value = True\n update_useractivity_cache(user.id, campaign.id, \"transcribe\")\n self.assertEqual(mock_update.call_count, 1)\n mock_update.assert_called_with(user.id, campaign.id, \"transcribe\")\n self.assertEqual(mock_delete.call_count, 1)\n mock_delete.assert_called_with(\"userprofileactivity_cache_lock\")\n\n update_useractivity_cache(user.id, campaign.id, \"review\")\n self.assertEqual(mock_update.call_count, 2)\n mock_update.assert_called_with(user.id, campaign.id, \"review\")\n self.assertEqual(mock_delete.call_count, 2)\n mock_delete.assert_called_with(\"userprofileactivity_cache_lock\")\n\n def test_populate_active_campaign_counts_computes_user_and_anon_rows(self):\n camp = create_campaign(slug=\"ua-camp-a\")\n proj = create_project(campaign=camp, slug=\"ua-proj-a\")\n item = create_item(project=proj, item_id=\"ua-item-a\")\n a1 = create_asset(item=item, slug=\"ua-a1\", sequence=1)\n a2 = create_asset(item=item, slug=\"ua-a2\", sequence=2)\n\n u1 = self.create_test_user(\"ua-u1\")\n u2 = self.create_test_user(\"ua-u2\")\n anon = get_anonymous_user()\n\n create_transcription(asset=a1, user=u1, reviewed_by=u2)\n create_transcription(asset=a2, user=u2, reviewed_by=u1)\n create_transcription(asset=a1, user=anon)\n create_transcription(asset=a2, user=u2, reviewed_by=anon)\n\n t1 = create_tag(value=\"ua-t1\")\n t2 = create_tag(value=\"ua-t2\")\n create_tag_collection(tag=t1, asset=a1, user=u1)\n create_tag_collection(tag=t2, asset=a2, user=u1)\n create_tag_collection(tag=t1, asset=a2, user=anon)\n\n populate_active_campaign_counts.run()\n\n rows = UserProfileActivity.objects.filter(campaign=camp)\n self.assertEqual(rows.count(), 3)\n\n r_u1 = rows.get(user=u1)\n r_u2 = rows.get(user=u2)\n r_an = rows.get(user=anon)\n\n self.assertEqual(r_u1.asset_count, 2)\n self.assertEqual(r_u1.asset_tag_count, 2)\n self.assertEqual(r_u1.transcribe_count, 1)\n self.assertEqual(r_u1.review_count, 1)\n\n self.assertEqual(r_u2.asset_count, 2)\n self.assertEqual(r_u2.asset_tag_count, 0)\n self.assertEqual(r_u2.transcribe_count, 2)\n self.assertEqual(r_u2.review_count, 1)\n\n self.assertEqual(r_an.asset_count, 2)\n self.assertEqual(r_an.asset_tag_count, 1)\n self.assertEqual(r_an.transcribe_count, 1)\n self.assertEqual(r_an.review_count, 1)\n\n def test_populate_completed_campaign_counts_processes_non_active_only(self):\n active = create_campaign(slug=\"ua-act-1\")\n p1 = create_project(campaign=active, slug=\"ua-act-proj\")\n it1 = create_item(project=p1, item_id=\"ua-act-item\")\n a_act = create_asset(item=it1, slug=\"ua-act-a\")\n u_act = self.create_test_user(\"ua-act-u\")\n create_transcription(asset=a_act, user=u_act)\n\n retired = create_campaign(slug=\"ua-ret-1\", status=Campaign.Status.RETIRED)\n p2 = create_project(campaign=retired, slug=\"ua-ret-proj\")\n it2 = create_item(project=p2, item_id=\"ua-ret-item\")\n a_ret = create_asset(item=it2, slug=\"ua-ret-a\")\n u_ret = self.create_test_user(\"ua-ret-u\")\n create_transcription(asset=a_ret, user=u_ret)\n\n populate_completed_campaign_counts.run()\n\n self.assertFalse(UserProfileActivity.objects.filter(campaign=active).exists())\n self.assertTrue(UserProfileActivity.objects.filter(campaign=retired).exists())\n\n def test_update_useractivity_cache_lock_max_retries_sends_email(self):\n with (\n mock.patch(\"django.core.cache.cache.add\", return_value=False),\n mock.patch.object(update_useractivity_cache, \"max_retries\", 0, create=True),\n mock.patch(\"concordia.tasks.useractivity.send_mail\") as m_send,\n mock.patch(\"concordia.logging.ConcordiaLogger.warning\") as m_warn,\n mock.patch(\"concordia.logging.ConcordiaLogger.exception\") as m_exc,\n mock.patch(\"concordia.tasks.useractivity.logger.error\") as m_err,\n ):\n with self.assertRaises(CacheLockedError):\n update_useractivity_cache.run(\n self.user.id, self.campaign.id, \"transcribe\"\n )\n\n # Structured logs were emitted\n self.assertTrue(m_warn.called)\n self.assertTrue(m_exc.called)\n\n # Email sent with expected subject\n self.assertTrue(m_send.called)\n sent_args, sent_kwargs = m_send.call_args\n self.assertEqual(\n sent_args[0],\n \"Task update_useractivity_cache failed: cache is locked.\",\n )\n # Unstructured error log emitted\n self.assertTrue(m_err.called)\n\n def test_update_useractivity_cache_update_exception_releases_lock(self):\n with (\n mock.patch(\"concordia.tasks.useractivity.cache.add\", return_value=True),\n mock.patch(\"concordia.tasks.useractivity.cache.delete\") as m_del,\n mock.patch(\n \"concordia.tasks.useractivity._update_useractivity_cache\",\n side_effect=RuntimeError(\"boom\"),\n ),\n mock.patch(\"concordia.tasks.useractivity.send_mail\") as m_mail,\n ):\n with self.assertRaises(RuntimeError):\n update_useractivity_cache.run(self.user.id, self.campaign.id, \"review\")\n\n m_del.assert_called_once_with(\"userprofileactivity_cache_lock\")\n m_mail.assert_not_called()\n\n\nclass UpdateUserprofileactivityFromCacheTestCase(CreateTestUsers, TestCase):\n def setUp(self):\n cache.clear()\n self.user = self.create_test_user()\n self.campaign = create_campaign()\n self.key = f\"userprofileactivity_{self.campaign.pk}\"\n\n @mock.patch(\"concordia.tasks.useractivity.update_userprofileactivity_table\")\n def test_no_updates(self, mock_update_table):\n cache.set(self.key, None)\n with mock.patch(\"concordia.logging.ConcordiaLogger.debug\") as mock_debug:\n update_userprofileactivity_from_cache()\n self.assertEqual(mock_debug.call_count, 2)\n mock_debug.assert_called_with(\n \"Cache contained no updates for key. Skipping\",\n event_code=\"update_userprofileactivity_from_cache_no_updates\",\n key=self.key,\n )\n self.assertEqual(mock_update_table.call_count, 0)\n\n @mock.patch(\"concordia.tasks.useractivity.update_userprofileactivity_table\")\n def test_update(self, mock_update_table):\n cache.set(self.key, {self.user.pk: (1, 0)})\n update_userprofileactivity_from_cache()\n self.assertEqual(mock_update_table.call_count, 2)\n mock_update_table.assert_has_calls(\n [\n mock.call(self.user, self.campaign.id, \"transcribe_count\", 1),\n mock.call(self.user, self.campaign.id, \"review_count\", 0),\n ]\n )\n self.assertIsNone(cache.get(self.key))\n"
},
{
"path": "concordia/tests/test_tasks_visualizations.py",
"content": "from datetime import timedelta\nfrom unittest import mock\n\nfrom django.core.cache import caches\nfrom django.test import TestCase, override_settings\nfrom django.utils import timezone\n\nfrom concordia.models import Campaign, SiteReport, TranscriptionStatus\nfrom concordia.tasks.visualizations import (\n populate_asset_status_visualization_cache,\n populate_daily_activity_visualization_cache,\n)\n\nfrom .utils import create_asset, create_campaign, create_item, create_project\n\n\n@override_settings(\n CACHES={\n \"default\": {\n \"BACKEND\": \"django.core.cache.backends.locmem.LocMemCache\",\n },\n \"visualization_cache\": {\n \"BACKEND\": \"django.core.cache.backends.locmem.LocMemCache\",\n },\n }\n)\nclass VisualizationCacheTasksTests(TestCase):\n class _UploadFailed(Exception):\n pass\n\n def setUp(self):\n self.cache = caches[\"visualization_cache\"]\n self.cache.clear()\n\n def test_populate_asset_status_visualization_cache(self):\n c1 = create_campaign(status=Campaign.Status.ACTIVE, title=\"Alpha\")\n c2 = create_campaign(status=Campaign.Status.ACTIVE, title=\"Beta\")\n p1 = create_project(campaign=c1)\n i1 = create_item(project=p1)\n p2 = create_project(campaign=c2)\n i2 = create_item(project=p2)\n create_asset(item=i1, transcription_status=TranscriptionStatus.NOT_STARTED)\n create_asset(\n item=i2,\n slug=\"test-asset-2\",\n transcription_status=TranscriptionStatus.IN_PROGRESS,\n )\n create_asset(\n item=i2,\n slug=\"test-asset-3\",\n transcription_status=TranscriptionStatus.SUBMITTED,\n )\n create_asset(\n item=i2,\n slug=\"test-asset-4\",\n transcription_status=TranscriptionStatus.COMPLETED,\n )\n\n populate_asset_status_visualization_cache.run()\n\n overview = self.cache.get(\"asset-status-overview\")\n expected_labels = [\n TranscriptionStatus.CHOICE_MAP[key]\n for key, _ in TranscriptionStatus.CHOICES\n ]\n self.assertEqual(overview[\"status_labels\"], expected_labels)\n # Totals: 1 not_started, 1 in_progress, 1 submitted, 1 completed\n self.assertEqual(overview[\"total_counts\"], [1, 1, 1, 1])\n\n def test_populate_daily_activity_visualization_cache(self):\n today = timezone.localdate()\n date1 = today - timedelta(days=2)\n date2 = today - timedelta(days=1)\n\n sr1 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n transcriptions_saved=5,\n daily_review_actions=1,\n )\n sr2 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n transcriptions_saved=10,\n daily_review_actions=2,\n )\n # Set specific created_on dates directly in DB\n SiteReport.objects.filter(pk=sr1.pk).update(created_on=date1)\n SiteReport.objects.filter(pk=sr2.pk).update(created_on=date2)\n\n populate_daily_activity_visualization_cache.run()\n\n result = self.cache.get(\"daily-transcription-activity-last-28-days\")\n self.assertIsNotNone(result)\n\n expected_labels = [(date2 - timedelta(days=1)), date2]\n expected_labels = [d.strftime(\"%Y-%m-%d\") for d in expected_labels]\n\n # Extract the two datasets\n datasets = result[\"transcription_datasets\"]\n self.assertEqual(len(datasets), 2)\n trans = next(ds for ds in datasets if ds[\"label\"] == \"Transcriptions\")\n reviews = next(ds for ds in datasets if ds[\"label\"] == \"Reviews\")\n\n # transcriptions = 5 on date1, 10 - 5 = 5 on date2\n # reviews = 1 on date1, 2 on date2\n self.assertEqual(trans[\"data\"][-2:], [5, 5]) # last two days in the data range\n self.assertEqual(reviews[\"data\"][-2:], [1, 2])\n\n def test_negative_daily_saved_clamps_to_zero(self):\n today = timezone.localdate()\n date1 = today - timedelta(days=2)\n date2 = today - timedelta(days=1)\n\n sr1 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n transcriptions_saved=10,\n daily_review_actions=0,\n )\n sr2 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n transcriptions_saved=5, # decreased total, which should not happen\n daily_review_actions=0,\n )\n SiteReport.objects.filter(pk=sr1.pk).update(created_on=date1)\n SiteReport.objects.filter(pk=sr2.pk).update(created_on=date2)\n\n populate_daily_activity_visualization_cache.run()\n\n result = self.cache.get(\"daily-transcription-activity-last-28-days\")\n self.assertIsNotNone(result)\n\n datasets = result[\"transcription_datasets\"]\n trans = next(ds for ds in datasets if ds[\"label\"] == \"Transcriptions\")\n\n # Should clamp the second day to 0\n self.assertEqual(trans[\"data\"][-2:], [10, 0])\n\n def test_asset_status_unchanged_skips_upload_and_cache_update(self):\n campaign = create_campaign(status=Campaign.Status.ACTIVE, title=\"Only\")\n project = create_project(campaign=campaign)\n item = create_item(project=project)\n create_asset(item=item, transcription_status=TranscriptionStatus.NOT_STARTED)\n create_asset(\n item=item, slug=\"a2\", transcription_status=TranscriptionStatus.IN_PROGRESS\n )\n create_asset(\n item=item, slug=\"a3\", transcription_status=TranscriptionStatus.SUBMITTED\n )\n create_asset(\n item=item, slug=\"a4\", transcription_status=TranscriptionStatus.COMPLETED\n )\n\n expected_counts = [1, 1, 1, 1]\n\n existing_payload = {\n \"status_labels\": [\n TranscriptionStatus.CHOICE_MAP[key]\n for key, _ in TranscriptionStatus.CHOICES\n ],\n \"total_counts\": expected_counts,\n \"csv_url\": \"https://old.example/asset-status.csv\",\n }\n self.cache.set(\"asset-status-overview\", existing_payload, None)\n\n with (\n mock.patch(\n \"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save\"\n ) as mock_save,\n mock.patch(\"concordia.tasks.visualizations.structured_logger\") as mock_log,\n ):\n populate_asset_status_visualization_cache.run()\n\n mock_save.assert_not_called()\n # Cache should remain as-is\n self.assertEqual(self.cache.get(\"asset-status-overview\"), existing_payload)\n # Logged unchanged\n self.assertTrue(mock_log.info.called)\n self.assertEqual(\n mock_log.info.call_args.kwargs.get(\"event_code\"),\n \"asset_status_vis_unchanged\",\n )\n\n def test_asset_status_upload_failure_with_prior_url_falls_back(self):\n campaign = create_campaign(status=Campaign.Status.ACTIVE, title=\"Only\")\n project = create_project(campaign=campaign)\n item = create_item(project=project)\n create_asset(item=item, transcription_status=TranscriptionStatus.NOT_STARTED)\n\n # Ensure \"existing\" differs so code takes the non-unchanged path\n self.cache.set(\n \"asset-status-overview\",\n {\n \"status_labels\": [],\n \"total_counts\": [0, 0, 0, 0],\n \"csv_url\": \"https://old.example/asset-status.csv\",\n },\n None,\n )\n\n with (\n mock.patch(\n \"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save\",\n side_effect=self._UploadFailed(\"test exception\"),\n ),\n mock.patch(\"concordia.tasks.visualizations.structured_logger\") as mock_log,\n ):\n # Should not raise because we have a prior CSV URL to fall back to\n populate_asset_status_visualization_cache.run()\n\n updated = self.cache.get(\"asset-status-overview\")\n # Counts should reflect the new data (1 in NOT_STARTED; others 0)\n expected = [\n 1 if key == TranscriptionStatus.NOT_STARTED else 0\n for key, _ in TranscriptionStatus.CHOICES\n ]\n self.assertEqual(updated[\"total_counts\"], expected)\n # URL should remain the old one\n self.assertEqual(updated[\"csv_url\"], \"https://old.example/asset-status.csv\")\n\n # Logged exception with the non-missing-url code\n self.assertTrue(mock_log.exception.called)\n self.assertEqual(\n mock_log.exception.call_args.kwargs.get(\"event_code\"),\n \"asset_status_vis_csv_error\",\n )\n\n def test_asset_status_upload_failure_without_prior_url_raises(self):\n campaign = create_campaign(status=Campaign.Status.ACTIVE, title=\"Only\")\n project = create_project(campaign=campaign)\n item = create_item(project=project)\n create_asset(item=item, transcription_status=TranscriptionStatus.NOT_STARTED)\n\n # No existing cache entry, so no prior URL\n with (\n mock.patch(\n \"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save\",\n side_effect=self._UploadFailed(\"test exception\"),\n ),\n mock.patch(\"concordia.tasks.visualizations.structured_logger\") as mock_log,\n ):\n with self.assertRaises(self._UploadFailed):\n populate_asset_status_visualization_cache.run()\n\n self.assertTrue(mock_log.exception.called)\n self.assertEqual(\n mock_log.exception.call_args.kwargs.get(\"event_code\"),\n \"asset_status_vis_csv_missing_url_error\",\n )\n\n def test_daily_activity_unchanged_skips_upload_and_cache_update(self):\n # With no SiteReports, both series are 28 zeros; pre-populate matching cache\n zeros = [0] * 28\n existing = {\n \"labels\": [], # labels do not matter for the dedupe\n \"transcription_datasets\": [\n {\"label\": \"Transcriptions\", \"data\": zeros},\n {\"label\": \"Reviews\", \"data\": zeros},\n ],\n \"csv_url\": \"https://old.example/daily.csv\",\n }\n self.cache.set(\"daily-transcription-activity-last-28-days\", existing, None)\n\n with (\n mock.patch(\n \"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save\"\n ) as mock_save,\n mock.patch(\"concordia.tasks.visualizations.structured_logger\") as mock_log,\n ):\n populate_daily_activity_visualization_cache.run()\n\n mock_save.assert_not_called()\n self.assertEqual(\n self.cache.get(\"daily-transcription-activity-last-28-days\"), existing\n )\n self.assertTrue(mock_log.info.called)\n self.assertEqual(\n mock_log.info.call_args.kwargs.get(\"event_code\"),\n \"daily_activity_vis_unchanged\",\n )\n\n def test_daily_activity_upload_failure_with_prior_url_falls_back(self):\n # Build reports so new data will not be all zeros (ensures \"changed\" path)\n today = timezone.localdate()\n date1 = today - timedelta(days=2)\n date2 = today - timedelta(days=1)\n sr1 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n transcriptions_saved=3,\n daily_review_actions=1,\n )\n sr2 = SiteReport.objects.create(\n report_name=SiteReport.ReportName.TOTAL,\n transcriptions_saved=5,\n daily_review_actions=2,\n )\n SiteReport.objects.filter(pk=sr1.pk).update(created_on=date1)\n SiteReport.objects.filter(pk=sr2.pk).update(created_on=date2)\n\n # Prior cache with different series and a CSV URL to fall back to\n self.cache.set(\n \"daily-transcription-activity-last-28-days\",\n {\n \"labels\": [],\n \"transcription_datasets\": [\n {\"label\": \"Transcriptions\", \"data\": [0] * 28},\n {\"label\": \"Reviews\", \"data\": [0] * 28},\n ],\n \"csv_url\": \"https://old.example/daily.csv\",\n },\n None,\n )\n\n with (\n mock.patch(\n \"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save\",\n side_effect=self._UploadFailed(\"test exception\"),\n ),\n mock.patch(\"concordia.tasks.visualizations.structured_logger\") as mock_log,\n ):\n # Should not raise because we have a prior CSV URL\n populate_daily_activity_visualization_cache.run()\n\n updated = self.cache.get(\"daily-transcription-activity-last-28-days\")\n self.assertIsNotNone(updated)\n # Still using the old URL\n self.assertEqual(updated[\"csv_url\"], \"https://old.example/daily.csv\")\n # Logged exception with the non-missing-url code\n self.assertTrue(mock_log.exception.called)\n self.assertEqual(\n mock_log.exception.call_args.kwargs.get(\"event_code\"),\n \"daily_activity_vis_csv_error\",\n )\n\n def test_daily_activity_upload_failure_without_prior_url_raises(self):\n # No existing cache entry -> csv_url is None\n with (\n mock.patch(\n \"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save\",\n side_effect=self._UploadFailed(\"test exception\"),\n ),\n mock.patch(\"concordia.tasks.visualizations.structured_logger\") as mock_log,\n ):\n with self.assertRaises(self._UploadFailed):\n populate_daily_activity_visualization_cache.run()\n\n self.assertTrue(mock_log.exception.called)\n self.assertEqual(\n mock_log.exception.call_args.kwargs.get(\"event_code\"),\n \"daily_activity_vis_csv_missing_url_error\",\n )\n"
},
{
"path": "concordia/tests/test_templatetags.py",
"content": "from django.http import QueryDict\nfrom django.template import Context, Template\nfrom django.templatetags.static import static\nfrom django.test import TestCase, override_settings\nfrom django.utils.html import escape, format_html\n\nfrom concordia.models import TranscriptionStatus\nfrom concordia.templatetags.concordia_filtering_tags import transcription_status_filters\nfrom concordia.templatetags.concordia_text_tags import reprchar\nfrom concordia.templatetags.custom_math import multiply\nfrom concordia.templatetags.reject_filter import reject\nfrom concordia.templatetags.truncation import (\n WordBreakTruncator,\n truncatechars_on_word_break,\n)\nfrom concordia.templatetags.visualization import concordia_visualization\n\n\nclass TestTemplateTags(TestCase):\n def test_truncatechars_on_word_break(self):\n test_string = \"Lorem ipsum \\u0317 dolor sit amet, consectetur adipiscing elit\"\n\n self.assertEqual(truncatechars_on_word_break(test_string, 0), \"[β¦]\")\n self.assertEqual(truncatechars_on_word_break(test_string, 1), \"[β¦]\")\n self.assertEqual(truncatechars_on_word_break(test_string, 10), \"Lorem[β¦]\")\n self.assertEqual(\n truncatechars_on_word_break(test_string, 30),\n \"Lorem ipsum \\u0317 dolor sit[β¦]\",\n )\n self.assertEqual(truncatechars_on_word_break(test_string, 1000), test_string)\n self.assertEqual(\n truncatechars_on_word_break(test_string, \"badvalue\"), test_string\n )\n\n self.assertEqual(\n WordBreakTruncator(test_string).word_break(30, \"[\\u0317]\"),\n \"Lorem ipsum \\u0317 dolor sit[\\u0317]\",\n )\n\n def test_multiply(self):\n self.assertEqual(multiply(5, 5), 5 * 5)\n self.assertEqual(multiply(0, 5), 0 * 5)\n self.assertEqual(multiply(1, 2), 1 * 2)\n\n def test_transcription_status_filters(self):\n status_counts = []\n for choice in TranscriptionStatus.CHOICES:\n status_counts.append((choice, 0, 1))\n\n transcription_status_filters(status_counts, \"\")\n transcription_status_filters(status_counts, \"\", reversed_order=True)\n transcription_status_filters(status_counts, TranscriptionStatus.CHOICES[0][0])\n\n def test_qs_alter(self):\n base_template = \"{% load concordia_querystring %}\"\n\n out = Template(\n base_template + \"{% qs_alter 'bar=baz&baz=taz' foo='bar' %}\"\n ).render(Context())\n self.assertEqual(out, \"bar=baz&baz=taz&foo=bar\")\n\n data = QueryDict(\"bar=baz&baz=taz&bar=foo\")\n out = Template(base_template + \"{% qs_alter data foo='bar' %}\").render(\n Context({\"data\": data})\n )\n self.assertEqual(out, \"bar=baz&bar=foo&baz=taz&foo=bar\")\n\n out = Template(base_template + \"{% qs_alter data delete:bar %}\").render(\n Context({\"data\": data})\n )\n self.assertEqual(out, \"baz=taz\")\n\n out = Template(base_template + \"{% qs_alter data delete:taz %}\").render(\n Context({\"data\": data})\n )\n self.assertEqual(out, \"bar=baz&bar=foo&baz=taz\")\n\n out = Template(\n base_template + \"{% qs_alter data delete_value:\\\"bar\\\",'foo' %}\"\n ).render(Context({\"data\": data}))\n self.assertEqual(out, \"bar=baz&baz=taz\")\n\n out = Template(\n base_template + \"{% qs_alter data delete_value:'bar','taz' %}\"\n ).render(Context({\"data\": data}))\n self.assertEqual(out, \"bar=baz&bar=foo&baz=taz\")\n\n out = Template(\n base_template + \"{% qs_alter data foo='bar' as new_data %}\" \"{{ new_data }}\"\n ).render(Context({\"data\": data}))\n self.assertEqual(out, \"bar=baz&bar=foo&baz=taz&foo=bar\")\n\n # Test add_if_missing when the key already exists (should not overwrite)\n out = Template(\n base_template + \"{% qs_alter data add_if_missing:bar='newvalue' %}\"\n ).render(Context({\"data\": data}))\n self.assertEqual(out, \"bar=baz&bar=foo&baz=taz\")\n\n def test_reprchar_variants(self):\n cases = [\n (\"A\", \"A\"),\n (\"\\n\", \"\\\\n\"),\n (\"\\x00\", \"\\\\x00\"),\n (\"\\u200b\", \"\\\\u200b\"),\n (\"\\\\\", \"\\\\\\\\\"),\n ]\n for ch, expected in cases:\n self.assertEqual(reprchar(ch), expected)\n\n\nclass RejectFilterTests(TestCase):\n def test_returns_input_when_falsy(self):\n self.assertEqual(reject(\"\", \"x\"), \"\")\n self.assertEqual(reject([], \"x\"), [])\n self.assertIsNone(reject(None, \"x\"))\n self.assertEqual(reject((), \"x\"), ())\n\n def test_string_single_reject(self):\n self.assertEqual(\n reject(\"error warn marked-safe\", \"marked-safe\"),\n \"error warn\",\n )\n\n def test_string_multiple_rejects(self):\n self.assertEqual(\n reject(\"error warn marked-safe\", \"marked-safe,warn\"),\n \"error\",\n )\n\n def test_string_no_match(self):\n self.assertEqual(reject(\"one two\", \"three\"), \"one two\")\n\n def test_string_empty_args(self):\n self.assertEqual(reject(\"one two\", \"\"), \"one two\")\n\n def test_string_whitespace_split_and_join(self):\n self.assertEqual(reject(\"a b\\tc\", \"b\"), \"a c\")\n\n def test_string_case_sensitivity(self):\n self.assertEqual(reject(\"A a\", \"a\"), \"A\")\n\n def test_iterable_list(self):\n self.assertEqual(\n reject([\"ok\", \"deprecated\", \"x\", \"hidden\"], \"deprecated,hidden\"),\n [\"ok\", \"x\"],\n )\n\n def test_iterable_tuple_and_duplicates(self):\n self.assertEqual(reject((\"a\", \"b\", \"c\", \"b\"), \"b\"), [\"a\", \"c\"])\n\n def test_iterable_no_match(self):\n self.assertEqual(reject([\"one\", \"two\"], \"three\"), [\"one\", \"two\"])\n\n def test_iterable_empty_args(self):\n self.assertEqual(reject([\"one\", \"two\"], \"\"), [\"one\", \"two\"])\n\n\n@override_settings(\n STORAGES={\n \"default\": {\"BACKEND\": \"django.core.files.storage.FileSystemStorage\"},\n \"staticfiles\": {\n \"BACKEND\": \"django.contrib.staticfiles.storage.StaticFilesStorage\",\n },\n },\n STATICFILES_STORAGE=\"django.contrib.staticfiles.storage.StaticFilesStorage\",\n)\nclass VisualizationTagsTests(TestCase):\n def test_without_attrs_renders_section_and_script(self):\n # No attributes: should render a plain and matching '\n script_src = static(f\"js/visualizations/{name}.js\")\n script_html = format_html(\n '', script_src\n )\n result = concordia_visualization(name) + script_html\n\n # The id attribute must have the name escaped\n escaped_id = escape(name)\n self.assertIn(f'id=\"{escaped_id}\"', result)\n\n # The script src must also be escaped\n raw_src = static(f\"js/visualizations/{name}.js\")\n escaped_src = escape(raw_src)\n self.assertIn(f'src=\"{escaped_src}\"', result)\n"
},
{
"path": "concordia/tests/test_top_level_views.py",
"content": "\"\"\"\nTests for for the top-level & βCMSβ views\n\"\"\"\n\nfrom unittest.mock import patch\n\nfrom django.core.cache import cache\nfrom django.test import RequestFactory, TestCase\nfrom django.urls import reverse\nfrom maintenance_mode.core import get_maintenance_mode, set_maintenance_mode\n\nfrom concordia.models import (\n Banner,\n CarouselSlide,\n Guide,\n OverlayPosition,\n SimplePage,\n SiteReport,\n)\nfrom concordia.views.simple_pages import simple_page\n\nfrom .utils import (\n CacheControlAssertions,\n CreateTestUsers,\n JSONAssertMixin,\n create_guide,\n create_site_report,\n)\n\n\nclass TopLevelViewTests(\n JSONAssertMixin, CreateTestUsers, CacheControlAssertions, TestCase\n):\n def setUp(self):\n cache.clear()\n\n def tearDown(self):\n cache.clear()\n\n def test_healthz(self):\n data = self.assertValidJSON(self.client.get(\"/healthz\"))\n\n for k in (\n \"current_time\",\n \"load_average\",\n \"debug\",\n \"database_has_data\",\n \"application_version\",\n ):\n self.assertIn(k, data)\n\n def test_homepage(self):\n response = self.client.get(reverse(\"homepage\"))\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(response, \"home.html\")\n\n banner = Banner.objects.create(\n slug=\"test-banner\", text=\"Test Banner\", active=True\n )\n response = self.client.get(reverse(\"homepage\"))\n context = response.context\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(response, \"home.html\")\n self.assertIn(\"banner\", context)\n self.assertEqual(context[\"banner\"].text, banner.text)\n banner.delete()\n\n slide = CarouselSlide.objects.create(\n published=True,\n overlay_position=OverlayPosition.LEFT,\n headline=\"Test Headline\",\n )\n response = self.client.get(reverse(\"homepage\"))\n context = response.context\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(response, \"home.html\")\n self.assertIn(\"firstslide\", context)\n self.assertEqual(context[\"firstslide\"].headline, slide.headline)\n slide.delete()\n\n def test_contact_us_redirect(self):\n response = self.client.get(reverse(\"contact\"))\n\n self.assertEqual(response.status_code, 302)\n self.assertEqual(response[\"Location\"], \"https://ask.loc.gov/crowd\")\n\n def test_simple_page(self):\n s = SimplePage.objects.create(\n title=\"Get Started 123\",\n body=\"not the real body\",\n path=reverse(\"welcome-guide\"),\n )\n\n s2 = SimplePage.objects.create(\n title=\"Get Started Spanish 123\",\n body=\"not the real spanish body\",\n path=reverse(\"welcome-guide-spanish\"),\n )\n\n resp = self.client.get(reverse(\"welcome-guide\"))\n self.assertEqual(200, resp.status_code)\n self.assertEqual(s.title, resp.context[\"title\"])\n self.assertEqual(\n [(reverse(\"welcome-guide\"), s.title)], resp.context[\"breadcrumbs\"]\n )\n self.assertEqual(resp.context[\"body\"], f\"{s.body}
\")\n\n request = RequestFactory().get(\"/\")\n resp = simple_page(request, path=reverse(\"welcome-guide\"))\n self.assertEqual(200, resp.status_code)\n\n resp = self.client.get(reverse(\"welcome-guide-spanish\"))\n self.assertEqual(200, resp.status_code)\n self.assertEqual(s2.title, resp.context[\"title\"])\n self.assertEqual(\"es\", resp.context[\"language_code\"])\n self.assertEqual(\n [(reverse(\"welcome-guide-spanish\"), s2.title)], resp.context[\"breadcrumbs\"]\n )\n self.assertEqual(resp.context[\"body\"], f\"{s2.body}
\")\n\n def test_nested_simple_page(self):\n Guide.objects.create(title=\"How to Tag\")\n l1 = SimplePage.objects.create(\n title=\"Get Started\",\n body=\"not the real body\",\n path=reverse(\"welcome-guide\"),\n )\n\n l2 = SimplePage.objects.create(\n title=\"How to Tag\",\n body=\"This is _not_ the real page\",\n path=reverse(\"how-to-tag\"),\n )\n\n resp = self.client.get(reverse(\"how-to-tag\"))\n self.assertEqual(200, resp.status_code)\n self.assertEqual(l2.title, resp.context[\"title\"])\n self.assertEqual(\n resp.context[\"breadcrumbs\"],\n [(reverse(\"welcome-guide\"), l1.title), (reverse(\"how-to-tag\"), l2.title)],\n )\n self.assertHTMLEqual(\n resp.context[\"body\"], \"This is not the real page
\"\n )\n\n create_guide(page=l1)\n resp = self.client.get(reverse(\"welcome-guide\"))\n self.assertEqual(200, resp.status_code)\n\n def test_simple_page_with_context(self):\n path = reverse(\"about\")\n page_body = (\n \"{{ assets_published}}
\"\n \"{{ campaigns_published }}
\"\n \"{{ assets_completed }}
\"\n \"{{ assets_waiting_review }}
\"\n \"{{ users_activated }}
\"\n )\n about_page = SimplePage.objects.create(\n title=\"About\",\n body=page_body,\n path=reverse(\"about\"),\n )\n\n # Test with no SiteReports\n response = self.client.get(path)\n context = response.context\n self.assertEqual(200, response.status_code)\n self.assertEqual(about_page.title, context[\"title\"])\n self.assertEqual([(path, about_page.title)], context[\"breadcrumbs\"])\n self.assertEqual(\n context[\"body\"], \"0
\\n0
\\n0
\\n0
\\n0
\"\n )\n\n # Test with only active SiteReport\n cache.clear()\n create_site_report(\n report_name=SiteReport.ReportName.TOTAL,\n campaigns_published=1,\n assets_published=1,\n assets_completed=1,\n assets_waiting_review=1,\n users_activated=1,\n )\n\n response = self.client.get(path)\n context = response.context\n self.assertEqual(\n context[\"body\"], \"1
\\n1
\\n1
\\n1
\\n1
\"\n )\n\n # Test with both SiteReports, but with cached values from above\n # So we should expect the retired SiteReport to not be included in data\n create_site_report(\n report_name=SiteReport.ReportName.RETIRED_TOTAL,\n assets_published=1,\n assets_completed=1,\n assets_waiting_review=1,\n )\n\n response = self.client.get(path)\n context = response.context\n self.assertEqual(\n context[\"body\"], \"1
\\n1
\\n1
\\n1
\\n1
\"\n )\n\n # Test without bad cached data\n cache.clear()\n response = self.client.get(path)\n context = response.context\n self.assertEqual(\n context[\"body\"], \"2
\\n1
\\n2
\\n2
\\n1
\"\n )\n\n\nclass HelpCenterRedirectTests(TestCase):\n def test_HelpCenterRedirectView(self):\n SimplePage.objects.create(\n title=\"Get Started Page\",\n body=\"Page Body\",\n path=\"/get-started/page/\",\n )\n\n self.assertRedirects(\n self.client.get(\"/help-center/page/\"), \"/get-started/page/\"\n )\n\n def test_HelpCenterSpanishRedirectView(self):\n SimplePage.objects.create(\n title=\"Get Started Page\",\n body=\"Page Body\",\n path=\"/get-started-esp/page-esp/\",\n )\n\n self.assertRedirects(\n self.client.get(\"/help-center/page-esp/\"), \"/get-started-esp/page-esp/\"\n )\n\n\nclass MaintenanceModeTests(TestCase, CreateTestUsers):\n def setUp(self):\n cache.clear()\n self.timestamp_value = 1\n self.user = None\n\n def tearDown(self):\n cache.clear()\n\n def test_maintenance_mode_off(self):\n self.user = self.create_super_user()\n self.login_user()\n set_maintenance_mode(True)\n\n with patch(\"concordia.views.maintenance_mode.time\") as mock:\n mock.return_value = self.timestamp_value\n self.assertRedirects(\n self.client.get(reverse(\"maintenance_mode_off\")),\n f\"/?t={self.timestamp_value}\",\n )\n self.assertEqual(get_maintenance_mode(), False)\n\n self.user = self.create_test_user()\n self.login_user()\n set_maintenance_mode(True)\n with patch(\"concordia.views.maintenance_mode.time\") as mock:\n mock.return_value = self.timestamp_value\n self.assertRedirects(\n self.client.get(reverse(\"maintenance_mode_off\")),\n f\"/?t={self.timestamp_value}\",\n target_status_code=503,\n )\n self.assertEqual(get_maintenance_mode(), True)\n\n def test_maintenance_mode_on_without_frontend(self):\n cache.set(\"maintenance_mode_frontend_available\", False, None)\n\n self.user = self.create_super_user()\n self.login_user()\n set_maintenance_mode(False)\n with patch(\"concordia.views.maintenance_mode.time\") as mock:\n mock.return_value = self.timestamp_value\n self.assertRedirects(\n self.client.get(reverse(\"maintenance_mode_on\")),\n f\"/?t={self.timestamp_value}\",\n target_status_code=503,\n )\n self.assertEqual(get_maintenance_mode(), True)\n\n self.user = self.create_test_user()\n self.login_user()\n set_maintenance_mode(False)\n with patch(\"concordia.views.maintenance_mode.time\") as mock:\n mock.return_value = self.timestamp_value\n self.assertRedirects(\n self.client.get(reverse(\"maintenance_mode_on\")),\n f\"/?t={self.timestamp_value}\",\n )\n self.assertEqual(get_maintenance_mode(), False)\n\n def test_maintenance_mode_on_with_frontend(self):\n cache.set(\"maintenance_mode_frontend_available\", True, None)\n\n self.user = self.create_super_user()\n self.login_user()\n set_maintenance_mode(False)\n with patch(\"concordia.views.maintenance_mode.time\") as mock:\n mock.return_value = self.timestamp_value\n self.assertRedirects(\n self.client.get(reverse(\"maintenance_mode_on\")),\n f\"/?t={self.timestamp_value}\",\n )\n self.assertEqual(get_maintenance_mode(), True)\n\n self.user = self.create_test_user()\n self.login_user()\n set_maintenance_mode(False)\n with patch(\"concordia.views.maintenance_mode.time\") as mock:\n mock.return_value = self.timestamp_value\n self.assertRedirects(\n self.client.get(reverse(\"maintenance_mode_on\")),\n f\"/?t={self.timestamp_value}\",\n )\n self.assertEqual(get_maintenance_mode(), False)\n\n def test_maintenance_mode_frontend_available(self):\n self.user = self.create_super_user()\n self.login_user()\n cache.set(\"maintenance_mode_frontend_available\", False, None)\n with patch(\"concordia.views.maintenance_mode.time\") as mock:\n mock.return_value = self.timestamp_value\n self.assertRedirects(\n self.client.get(reverse(\"maintenance_mode_frontend_available\")),\n f\"/?t={self.timestamp_value}\",\n )\n self.assertEqual(cache.get(\"maintenance_mode_frontend_available\"), True)\n\n self.user = self.create_test_user()\n self.login_user()\n cache.set(\"maintenance_mode_frontend_available\", False, None)\n with patch(\"concordia.views.maintenance_mode.time\") as mock:\n mock.return_value = self.timestamp_value\n self.assertRedirects(\n self.client.get(reverse(\"maintenance_mode_frontend_available\")),\n f\"/?t={self.timestamp_value}\",\n )\n self.assertEqual(cache.get(\"maintenance_mode_frontend_available\"), False)\n\n def test_maintenance_mode_frontend_unavailable(self):\n self.user = self.create_super_user()\n self.login_user()\n cache.set(\"maintenance_mode_frontend_available\", True, None)\n with patch(\"concordia.views.maintenance_mode.time\") as mock:\n mock.return_value = self.timestamp_value\n self.assertRedirects(\n self.client.get(reverse(\"maintenance_mode_frontend_unavailable\")),\n f\"/?t={self.timestamp_value}\",\n )\n self.assertEqual(cache.get(\"maintenance_mode_frontend_available\"), False)\n\n self.user = self.create_test_user()\n self.login_user()\n cache.set(\"maintenance_mode_frontend_available\", True, None)\n with patch(\"concordia.views.maintenance_mode.time\") as mock:\n mock.return_value = self.timestamp_value\n self.assertRedirects(\n self.client.get(reverse(\"maintenance_mode_frontend_unavailable\")),\n f\"/?t={self.timestamp_value}\",\n )\n self.assertEqual(cache.get(\"maintenance_mode_frontend_available\"), True)\n"
},
{
"path": "concordia/tests/test_utils_celery.py",
"content": "from types import SimpleNamespace\nfrom unittest import mock\n\nfrom django.test import TestCase\n\nfrom concordia.utils.celery import get_registered_task\n\n\nclass CeleryUtilsTests(TestCase):\n def test_get_registered_task_returns_task_from_registry(self):\n name = \"pkg.tasks.do_thing\"\n dummy_task = object()\n app = SimpleNamespace(tasks={name: dummy_task}, send_task=mock.Mock())\n\n with mock.patch(\n \"concordia.utils.celery.concordia_celery_app\",\n app,\n ):\n got = get_registered_task(name)\n\n self.assertIs(got, dummy_task)\n app.send_task.assert_not_called()\n\n def test_get_registered_task_raises_runtime_error_with_cause(self):\n name = \"pkg.tasks.missing\"\n app = SimpleNamespace(tasks={}, send_task=mock.Mock())\n\n with mock.patch(\n \"concordia.utils.celery.concordia_celery_app\",\n app,\n ):\n with self.assertRaises(RuntimeError) as ctx:\n get_registered_task(name)\n\n message = str(ctx.exception)\n self.assertIn(f\"Task {name} is not registered.\", message)\n self.assertIn(\"Did you typo it?\", message)\n self.assertIsInstance(ctx.exception.__cause__, KeyError)\n app.send_task.assert_not_called()\n"
},
{
"path": "concordia/tests/test_utils_logging.py",
"content": "from types import SimpleNamespace\n\nfrom django.test import TestCase\n\nfrom concordia.logging import get_logging_user_id\nfrom concordia.utils import get_anonymous_user\n\nfrom .utils import CreateTestUsers\n\n\nclass LoggingTests(CreateTestUsers, TestCase):\n def test_get_logging_user_id_authenticated_user(self):\n user = self.create_test_user()\n self.assertEqual(get_logging_user_id(user), str(user.id))\n\n def test_get_logging_user_id_anonymous_user(self):\n anon = get_anonymous_user()\n self.assertEqual(get_logging_user_id(anon), \"anonymous\")\n\n def test_get_logging_user_id_missing_auth_attribute(self):\n mock_user = object()\n self.assertEqual(get_logging_user_id(mock_user), \"anonymous\")\n\n def test_get_logging_user_id_authenticated_no_id(self):\n user = SimpleNamespace(is_authenticated=True, username=\"someuser\")\n self.assertEqual(get_logging_user_id(user), \"anonymous\")\n"
},
{
"path": "concordia/tests/test_utils_next_asset_reviewable_campaign.py",
"content": "from unittest.mock import MagicMock, patch\n\nfrom django.test import TestCase\nfrom django.utils.timezone import now\n\nfrom concordia.models import (\n AssetTranscriptionReservation,\n NextReviewableCampaignAsset,\n)\nfrom concordia.utils import get_anonymous_user\nfrom concordia.utils.next_asset import (\n find_new_reviewable_campaign_assets,\n find_next_reviewable_campaign_asset,\n find_reviewable_campaign_asset,\n)\nfrom concordia.utils.next_asset.reviewable.campaign import (\n _eligible_reviewable_base_qs,\n _find_reviewable_in_item,\n _find_reviewable_in_project,\n _next_seq_after,\n _reserved_asset_ids_subq,\n find_and_order_potential_reviewable_campaign_assets,\n find_invalid_next_reviewable_campaign_assets,\n)\n\nfrom .utils import (\n CreateTestUsers,\n create_asset,\n create_campaign,\n create_item,\n create_project,\n create_transcription,\n)\n\n\nclass NextReviewableCampaignAssetTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anon = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(sequence=1)\n self.asset2 = create_asset(\n item=self.asset1.item, sequence=2, slug=\"test-asset-2\"\n )\n self.campaign = self.asset1.campaign\n\n def test_find_new_reviewable_campaign_assets_filters_correctly(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n\n queryset = find_new_reviewable_campaign_assets(self.campaign, self.user)\n self.assertIn(self.asset1, queryset)\n\n def test_find_new_reviewable_campaign_assets_without_user(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n # Covers lines 28β30\n queryset = find_new_reviewable_campaign_assets(self.campaign, None)\n self.assertIn(self.asset1, queryset)\n\n def test_find_reviewable_campaign_asset_from_next_table(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n\n NextReviewableCampaignAsset.objects.create(\n asset=self.asset1,\n campaign=self.campaign,\n item=self.asset1.item,\n item_item_id=self.asset1.item.item_id,\n project=self.asset1.item.project,\n project_slug=self.asset1.item.project.slug,\n sequence=self.asset1.sequence,\n transcriber_ids=[],\n )\n\n asset = find_reviewable_campaign_asset(self.campaign, self.user)\n self.assertEqual(asset, self.asset1)\n\n @patch(\"concordia.utils.next_asset.reviewable.campaign.get_registered_task\")\n def test_find_reviewable_campaign_asset_falls_back_and_spawns_task(\n self, mock_get_task\n ):\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset = find_reviewable_campaign_asset(self.campaign, self.user)\n self.assertEqual(asset, self.asset2)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.campaign.get_registered_task\")\n def test_find_next_reviewable_campaign_asset_orders_and_falls_back(\n self, mock_get_task\n ):\n \"\"\"\n With short-circuiting: project-level returns the eligible asset\n and we do not spawn a task.\n \"\"\"\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(asset, self.asset1)\n # Short-circuit satisfied -> no cache fallback -> no task spawned\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.campaign.get_registered_task\")\n def test_find_next_reviewable_campaign_asset_when_next_asset_exists(\n self, mock_get_task\n ):\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n NextReviewableCampaignAsset.objects.create(\n asset=self.asset2,\n campaign=self.campaign,\n item=self.asset2.item,\n item_item_id=self.asset2.item.item_id,\n project=self.asset2.item.project,\n project_slug=self.asset2.item.project.slug,\n sequence=self.asset2.sequence,\n transcriber_ids=[],\n )\n\n asset = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=self.asset2.item.project.slug,\n item_id=self.asset2.item.item_id,\n original_asset_id=self.asset2.id - 1,\n )\n self.assertEqual(asset, self.asset2)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n def test_short_circuit_same_item_excludes_users_own_work(self):\n \"\"\"\n Reviewable item short-circuit must not return assets\n transcribed by the requesting user.\n \"\"\"\n # Two submitted in same item: one by self.user, one by anon\n mine = self.asset1\n other = self.asset2\n create_transcription(asset=mine, user=self.user, submitted=now())\n create_transcription(asset=other, user=self.anon, submitted=now())\n\n chosen = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=mine.item.project.slug,\n item_id=mine.item.item_id,\n original_asset_id=mine.id,\n )\n self.assertEqual(chosen, other)\n\n def test_item_short_circuit_reviewable_respects_after_sequence_and_reservations(\n self,\n ):\n \"\"\"\n Item reviewable short-circuit should choose next by sequence\n and skip reserved assets.\n \"\"\"\n # Three submitted in the same item (none by self.user)\n asset1 = self.asset1\n asset2 = self.asset2\n asset3 = create_asset(item=asset1.item, sequence=3, slug=\"rev-a3\")\n for asset in (asset1, asset2, asset3):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n\n # After asset1, pick asset2\n chosen = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=asset1.item.project.slug,\n item_id=asset1.item.item_id,\n original_asset_id=asset1.id,\n )\n self.assertEqual(chosen, asset2)\n\n # Reserve asset2, so should pick asset3\n AssetTranscriptionReservation.objects.create(\n asset=asset2, reservation_token=\"rv\" # nosec\n )\n chosen2 = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=asset1.item.project.slug,\n item_id=asset1.item.item_id,\n original_asset_id=asset1.id,\n )\n self.assertEqual(chosen2, asset3)\n\n def test_project_short_circuit_when_item_has_only_users_work(self):\n \"\"\"\n If the only SUBMITTED assets in the item were transcribed by the user,\n the function should fall back to other assets in the same project.\n \"\"\"\n # Item-level: only user's work\n create_transcription(asset=self.asset1, user=self.user, submitted=now())\n create_transcription(asset=self.asset2, user=self.user, submitted=now())\n\n # Project-level: someone else's work\n other_item = create_item(project=self.asset1.item.project, item_id=\"p2\")\n project_asset = create_asset(item=other_item, slug=\"rev-p-asset\")\n create_transcription(asset=project_asset, user=self.anon, submitted=now())\n\n chosen = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, project_asset)\n\n @patch(\"concordia.utils.next_asset.reviewable.campaign.get_registered_task\")\n def test_cache_excludes_user_and_triggers_spawn_task(self, mock_get_task):\n \"\"\"\n When the only cached asset is excluded via transcriber_ids containing the user,\n the function should skip cache, fall back to manual and spawn a populate task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Cached entry references the user's own work (excluded by contains)\n create_transcription(asset=self.asset1, user=self.user, submitted=now())\n NextReviewableCampaignAsset.objects.create(\n asset=self.asset1,\n campaign=self.campaign,\n item=self.asset1.item,\n item_item_id=self.asset1.item.item_id,\n project=self.asset1.item.project,\n project_slug=self.asset1.item.project.slug,\n sequence=self.asset1.sequence,\n transcriber_ids=[self.user.id],\n )\n\n # A valid reviewable exists elsewhere\n other = create_asset(item=self.asset1.item, sequence=3, slug=\"rev-other\")\n create_transcription(asset=other, user=self.anon, submitted=now())\n\n # Pass empty project/item to ensure we hit cache (and thus spawn task)\n chosen = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=None,\n )\n self.assertEqual(chosen, other)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n\nclass ReviewableCampaignInternalsTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anon = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(sequence=1, slug=\"rc-a1\")\n # same item, higher sequence\n self.asset2 = create_asset(item=self.asset1.item, sequence=2, slug=\"rc-a2\")\n self.campaign = self.asset1.campaign\n\n def test_reserved_asset_ids_subq_filters_to_campaign(self):\n AssetTranscriptionReservation.objects.create(\n asset=self.asset1, reservation_token=\"r1\" # nosec\n )\n other_campaign = create_campaign(slug=\"rc-camp-a\", title=\"rc-camp-a\")\n other_project = create_project(\n campaign=other_campaign, slug=\"rc-proj-a\", title=\"rc-proj-a\"\n )\n other_item = create_item(project=other_project, item_id=\"rc-other-item\")\n other_campaign_asset = create_asset(item=other_item, slug=\"rc-other-camp-a\")\n AssetTranscriptionReservation.objects.create(\n asset=other_campaign_asset, reservation_token=\"r2\" # nosec\n )\n\n id_set = set(\n _reserved_asset_ids_subq(self.campaign).values_list(\"asset_id\", flat=True)\n )\n self.assertIn(self.asset1.id, id_set)\n self.assertNotIn(other_campaign_asset.id, id_set)\n\n def test_eligible_reviewable_base_qs_excludes_user_and_requires_submitted(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.user, submitted=now())\n asset3 = create_asset(item=self.asset1.item, sequence=3, slug=\"rc-a3\")\n\n queryset_user = _eligible_reviewable_base_qs(self.campaign, user=self.user)\n self.assertIn(self.asset1, queryset_user)\n self.assertNotIn(self.asset2, queryset_user)\n self.assertNotIn(asset3, queryset_user)\n\n queryset_none = _eligible_reviewable_base_qs(self.campaign, user=None)\n self.assertIn(self.asset1, queryset_none)\n self.assertIn(self.asset2, queryset_none)\n self.assertNotIn(asset3, queryset_none)\n\n def test_next_seq_after_none_missing_and_valid(self):\n self.assertIsNone(_next_seq_after(None))\n self.assertIsNone(_next_seq_after(99999999))\n self.assertEqual(_next_seq_after(self.asset2.pk), self.asset2.sequence)\n\n def test_find_reviewable_in_item_after_none_returns_first(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n chosen = _find_reviewable_in_item(\n self.campaign,\n self.user,\n item_id=self.asset1.item.item_id,\n after_asset_pk=None,\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_find_reviewable_in_item_after_asset_in_other_item_ignores_gate(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"rc-other-item\"\n )\n other_asset = create_asset(item=other_item, slug=\"rc-other-asset\")\n create_transcription(asset=other_asset, user=self.anon, submitted=now())\n\n chosen = _find_reviewable_in_item(\n self.campaign,\n self.user,\n item_id=self.asset1.item.item_id,\n after_asset_pk=other_asset.id,\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_find_reviewable_in_item_after_asset_missing_ignores_gate(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n chosen = _find_reviewable_in_item(\n self.campaign,\n self.user,\n item_id=self.asset1.item.item_id,\n after_asset_pk=987654321,\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_find_reviewable_in_item_after_asset_sidc_ignores_gate(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n other_campaign = create_campaign(slug=\"rc-camp-b\", title=\"rc-camp-b\")\n other_project = create_project(\n campaign=other_campaign, slug=\"rc-proj-b\", title=\"rc-proj-b\"\n )\n other_item = create_item(\n project=other_project, item_id=self.asset1.item.item_id\n )\n other_campaign_asset = create_asset(item=other_item, slug=\"rc-cross-camp\")\n create_transcription(\n asset=other_campaign_asset, user=self.anon, submitted=now()\n )\n\n chosen = _find_reviewable_in_item(\n self.campaign,\n self.user,\n item_id=self.asset1.item.item_id,\n after_asset_pk=other_campaign_asset.id,\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_find_reviewable_in_project_orders_and_excludes_user(self):\n other_item = create_item(project=self.asset1.item.project, item_id=\"rc-p-item\")\n mine = create_asset(item=other_item, sequence=1, slug=\"rc-p-mine\")\n theirs = create_asset(item=other_item, sequence=2, slug=\"rc-p-theirs\")\n create_transcription(asset=mine, user=self.user, submitted=now())\n create_transcription(asset=theirs, user=self.anon, submitted=now())\n\n chosen = _find_reviewable_in_project(\n self.campaign,\n self.user,\n project_slug=self.asset1.item.project.slug,\n after_asset_pk=self.asset1.id,\n )\n self.assertEqual(chosen, theirs)\n\n def test_find_reviewable_in_project_returns_none_when_only_users_work(self):\n other_item = create_item(project=self.asset1.item.project, item_id=\"rc-p2\")\n mine = create_asset(item=other_item, sequence=1, slug=\"rc-p2-mine\")\n create_transcription(asset=mine, user=self.user, submitted=now())\n\n chosen = _find_reviewable_in_project(\n self.campaign,\n self.user,\n project_slug=self.asset1.item.project.slug,\n after_asset_pk=self.asset1.id,\n )\n self.assertIsNone(chosen)\n\n def test_find_new_reviewable_campaign_assets_excludes_reserved_and_next_table(\n self,\n ):\n asset_reserved = create_asset(\n item=self.asset1.item, sequence=3, slug=\"rc-a-res\"\n )\n asset_cached = create_asset(\n item=self.asset1.item, sequence=4, slug=\"rc-a-cached\"\n )\n for asset in (asset_reserved, asset_cached):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n\n AssetTranscriptionReservation.objects.create(\n asset=asset_reserved, reservation_token=\"rv\" # nosec\n )\n\n from concordia.models import NextReviewableCampaignAsset\n\n NextReviewableCampaignAsset.objects.create(\n asset=asset_cached,\n campaign=self.campaign,\n item=asset_cached.item,\n item_item_id=asset_cached.item.item_id,\n project=asset_cached.item.project,\n project_slug=asset_cached.item.project.slug,\n sequence=asset_cached.sequence,\n transcriber_ids=[],\n )\n\n queryset = find_new_reviewable_campaign_assets(self.campaign, self.user)\n self.assertNotIn(asset_reserved, queryset)\n self.assertNotIn(asset_cached, queryset)\n\n def test_find_and_order_potential_reviewable_campaign_assets_ordering(self):\n base_item = self.asset1.item\n\n same_item_next = create_asset(item=base_item, sequence=10, slug=\"rc-ci-next\")\n\n other_item_same_project = create_asset(\n item=create_item(project=base_item.project, item_id=\"rc-it-2\"),\n sequence=5,\n slug=\"rc-p-next\",\n )\n\n other_project = create_project(\n campaign=self.campaign, slug=\"rc-proj\", title=\"rc-proj\"\n )\n other_project_item = create_item(project=other_project, item_id=\"rc-it-3\")\n other_project_asset = create_asset(\n item=other_project_item, sequence=1, slug=\"rc-op\"\n )\n\n for asset in (same_item_next, other_item_same_project, other_project_asset):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n\n def cache_row(asset):\n return NextReviewableCampaignAsset.objects.create(\n asset=asset,\n campaign=self.campaign,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n sequence=asset.sequence,\n transcriber_ids=[],\n )\n\n cache_row(same_item_next)\n cache_row(other_item_same_project)\n cache_row(other_project_asset)\n\n after_primary_key = self.asset1.id\n\n ordered = find_and_order_potential_reviewable_campaign_assets(\n self.campaign,\n self.user,\n project_slug=base_item.project.slug,\n item_id=base_item.item_id,\n asset_pk=after_primary_key,\n ).values_list(\"asset_id\", flat=True)\n\n ordered = list(ordered)\n self.assertEqual(ordered[0], same_item_next.id)\n self.assertEqual(ordered[1], other_item_same_project.id)\n self.assertIn(other_project_asset.id, ordered[2:])\n\n @patch(\"concordia.utils.next_asset.reviewable.campaign.get_registered_task\")\n def test_find_reviewable_campaign_asset_no_eligible_spawns_task_and_returns_none(\n self, mock_get_task\n ):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset = find_reviewable_campaign_asset(self.campaign, self.user)\n self.assertIsNone(asset)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.campaign.get_registered_task\")\n def test_manual_fallback_orders_and_spawns_task(self, mock_get_task):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset_x = create_asset(item=self.asset1.item, sequence=7, slug=\"rc-mf-x\")\n asset_y = create_asset(item=self.asset1.item, sequence=8, slug=\"rc-mf-y\")\n create_transcription(asset=asset_x, user=self.anon, submitted=now())\n create_transcription(asset=asset_y, user=self.anon, submitted=now())\n\n chosen = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=asset_x.id, # makes asset_y the \"next\" by id\n )\n self.assertEqual(chosen, asset_y)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n def test_find_invalid_next_reviewable_campaign_assets_reserved_and_wrong_status(\n self,\n ):\n from concordia.models import NextReviewableCampaignAsset\n\n reserved_asset = create_asset(\n item=self.asset1.item, sequence=30, slug=\"rc-inv-res\"\n )\n create_transcription(asset=reserved_asset, user=self.anon, submitted=now())\n AssetTranscriptionReservation.objects.create(\n asset=reserved_asset, reservation_token=\"rv\" # nosec\n )\n NextReviewableCampaignAsset.objects.create(\n asset=reserved_asset,\n campaign=self.campaign,\n item=reserved_asset.item,\n item_item_id=reserved_asset.item.item_id,\n project=reserved_asset.item.project,\n project_slug=reserved_asset.item.project.slug,\n sequence=reserved_asset.sequence,\n transcriber_ids=[],\n )\n\n wrong_status_asset = create_asset(\n item=self.asset1.item, sequence=31, slug=\"rc-inv-wrong\"\n )\n create_transcription(asset=wrong_status_asset, user=self.anon) # IN_PROGRESS\n NextReviewableCampaignAsset.objects.create(\n asset=wrong_status_asset,\n campaign=self.campaign,\n item=wrong_status_asset.item,\n item_item_id=wrong_status_asset.item.item_id,\n project=wrong_status_asset.item.project,\n project_slug=wrong_status_asset.item.project.slug,\n sequence=wrong_status_asset.sequence,\n transcriber_ids=[],\n )\n\n invalid = list(\n find_invalid_next_reviewable_campaign_assets(self.campaign.id).values_list(\n \"asset_id\", flat=True\n )\n )\n self.assertIn(reserved_asset.id, invalid)\n self.assertIn(wrong_status_asset.id, invalid)\n\n def test_item_short_circuit_internal_applies_after_and_skips_reserved(self):\n asset1 = self.asset1\n asset2 = self.asset2\n asset3 = create_asset(item=asset1.item, sequence=3, slug=\"rc-int-a3\")\n for asset in (asset1, asset2, asset3):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n AssetTranscriptionReservation.objects.create(\n asset=asset2, reservation_token=\"rv-int\" # nosec\n )\n\n chosen = _find_reviewable_in_item(\n self.campaign,\n self.user,\n item_id=asset1.item.item_id,\n after_asset_pk=asset1.id,\n )\n self.assertEqual(chosen, asset3)\n\n def test_item_short_circuit_internal_excludes_users_own_work(self):\n mine = self.asset1\n other = self.asset2\n create_transcription(asset=mine, user=self.user, submitted=now())\n create_transcription(asset=other, user=self.anon, submitted=now())\n\n chosen = _find_reviewable_in_item(\n self.campaign,\n self.user,\n item_id=mine.item.item_id,\n after_asset_pk=None,\n )\n self.assertEqual(chosen, other)\n\n def test_project_short_circuit_internal_skips_reserved_first(self):\n project = self.asset1.item.project\n item2 = create_item(project=project, item_id=\"rc-proj-int\")\n first = create_asset(item=item2, sequence=1, slug=\"rc-proj-int-1\")\n second = create_asset(item=item2, sequence=2, slug=\"rc-proj-int-2\")\n for asset in (first, second):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n AssetTranscriptionReservation.objects.create(\n asset=first, reservation_token=\"rv-proj-int\" # nosec\n )\n\n chosen = _find_reviewable_in_project(\n self.campaign,\n self.user,\n project_slug=project.slug,\n after_asset_pk=self.asset1.id,\n )\n self.assertEqual(chosen, second)\n\n def test_order_potential_without_after_prefers_item_then_project(self):\n base_item = self.asset1.item\n\n same_item = create_asset(item=base_item, sequence=9, slug=\"rc-ci-none\")\n same_project = create_asset(\n item=create_item(project=base_item.project, item_id=\"rc-it-np\"),\n sequence=2,\n slug=\"rc-p-none\",\n )\n other_project = create_asset(\n item=create_item(\n project=create_project(\n campaign=self.campaign, slug=\"rc-proj-none\", title=\"rc-proj-none\"\n ),\n item_id=\"rc-it-op-none\",\n ),\n sequence=1,\n slug=\"rc-op-none\",\n )\n for asset in (same_item, same_project, other_project):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n\n NextReviewableCampaignAsset.objects.create(\n asset=same_item,\n campaign=self.campaign,\n item=same_item.item,\n item_item_id=same_item.item.item_id,\n project=same_item.item.project,\n project_slug=same_item.item.project.slug,\n sequence=same_item.sequence,\n transcriber_ids=[],\n )\n NextReviewableCampaignAsset.objects.create(\n asset=same_project,\n campaign=self.campaign,\n item=same_project.item,\n item_item_id=same_project.item.item_id,\n project=same_project.item.project,\n project_slug=same_project.item.project.slug,\n sequence=same_project.sequence,\n transcriber_ids=[],\n )\n NextReviewableCampaignAsset.objects.create(\n asset=other_project,\n campaign=self.campaign,\n item=other_project.item,\n item_item_id=other_project.item.item_id,\n project=other_project.item.project,\n project_slug=other_project.item.project.slug,\n sequence=other_project.sequence,\n transcriber_ids=[],\n )\n\n ordered = find_and_order_potential_reviewable_campaign_assets(\n self.campaign,\n self.user,\n project_slug=base_item.project.slug,\n item_id=base_item.item_id,\n asset_pk=None, # ensure next_asset==0\n ).values_list(\"asset_id\", flat=True)\n\n ordered = list(ordered)\n self.assertEqual(ordered[0], same_item.id)\n self.assertEqual(ordered[1], same_project.id)\n self.assertIn(other_project.id, ordered[2:])\n\n @patch(\"concordia.utils.next_asset.reviewable.campaign.get_registered_task\")\n def test_next_reviewable_manual_fallback_no_after_spawns_and_picks_lowest_seq(\n self, mock_get_task\n ):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset1 = self.asset1\n asset2 = self.asset2\n create_transcription(asset=asset1, user=self.anon, submitted=now())\n create_transcription(asset=asset2, user=self.anon, submitted=now())\n\n chosen = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=None, # triggers Value(0) annotation branch\n )\n self.assertEqual(chosen, asset1)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.campaign.get_registered_task\")\n def test_next_reviewable_manual_fallback_invalid_after_str(self, mock_get_task):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset1 = self.asset1\n asset2 = self.asset2\n create_transcription(asset=asset1, user=self.anon, submitted=now())\n create_transcription(asset=asset2, user=self.anon, submitted=now())\n\n chosen = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=\"not-an-int\",\n )\n self.assertEqual(chosen, asset1)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.campaign.get_registered_task\")\n def test_next_reviewable_cached_path_when_short_circuits_fail(self, mock_get_task):\n \"\"\"\n Provide item_id and project_slug so short-circuits run but fail\n (only user's SUBMITTED work in that scope), then ensure we use\n the cached table (no task spawned).\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n create_transcription(asset=self.asset1, user=self.user, submitted=now())\n create_transcription(asset=self.asset2, user=self.user, submitted=now())\n\n cached_project = create_project(\n campaign=self.campaign, slug=\"rc-cached-proj\", title=\"rc-cached-proj\"\n )\n cached_item = create_item(project=cached_project, item_id=\"rc-cached-item\")\n cached_asset = create_asset(item=cached_item, slug=\"rc-cached-asset\")\n create_transcription(asset=cached_asset, user=self.anon, submitted=now())\n\n NextReviewableCampaignAsset.objects.create(\n asset=cached_asset,\n campaign=self.campaign,\n item=cached_asset.item,\n item_item_id=cached_asset.item.item_id,\n project=cached_asset.item.project,\n project_slug=cached_asset.item.project.slug,\n sequence=cached_asset.sequence,\n transcriber_ids=[],\n )\n\n chosen = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, cached_asset)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.campaign.get_registered_task\")\n def test_next_reviewable_uses_cache_when_bypassing_short_circuits(\n self, mock_get_task\n ):\n \"\"\"\n Skip both short-circuits by passing blanks; ensure we return from\n the cache table directly (no task spawned).\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n cached_project = create_project(\n campaign=self.campaign,\n slug=\"rc-cached-proj-2\",\n title=\"rc-cached-proj-2\",\n )\n cached_item = create_item(project=cached_project, item_id=\"rc-cached-item-2\")\n cached_asset = create_asset(item=cached_item, slug=\"rc-cached-asset-2\")\n create_transcription(asset=cached_asset, user=self.anon, submitted=now())\n\n NextReviewableCampaignAsset.objects.create(\n asset=cached_asset,\n campaign=self.campaign,\n item=cached_asset.item,\n item_item_id=cached_asset.item.item_id,\n project=cached_asset.item.project,\n project_slug=cached_asset.item.project.slug,\n sequence=cached_asset.sequence,\n transcriber_ids=[],\n )\n\n chosen = find_next_reviewable_campaign_asset(\n self.campaign,\n self.user,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=None,\n )\n self.assertEqual(chosen, cached_asset)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n"
},
{
"path": "concordia/tests/test_utils_next_asset_reviewable_topic.py",
"content": "from unittest.mock import MagicMock, patch\n\nfrom django.test import TestCase\nfrom django.utils.timezone import now\n\nfrom concordia.models import (\n AssetTranscriptionReservation,\n NextReviewableTopicAsset,\n)\nfrom concordia.utils import get_anonymous_user\nfrom concordia.utils.next_asset import (\n find_new_reviewable_topic_assets,\n find_next_reviewable_topic_asset,\n find_reviewable_topic_asset,\n)\nfrom concordia.utils.next_asset.reviewable.topic import (\n _eligible_reviewable_base_qs as topic_eligible_reviewable_base_qs,\n)\nfrom concordia.utils.next_asset.reviewable.topic import (\n _find_reviewable_in_item as topic_find_reviewable_in_item,\n)\nfrom concordia.utils.next_asset.reviewable.topic import (\n _find_reviewable_in_project as topic_find_reviewable_in_project,\n)\nfrom concordia.utils.next_asset.reviewable.topic import (\n _next_seq_after as topic_next_seq_after,\n)\nfrom concordia.utils.next_asset.reviewable.topic import (\n _reserved_asset_ids_subq as topic_reserved_asset_ids_subq,\n)\nfrom concordia.utils.next_asset.reviewable.topic import (\n find_and_order_potential_reviewable_topic_assets,\n find_invalid_next_reviewable_topic_assets,\n)\nfrom concordia.utils.next_asset.reviewable.topic import (\n find_next_reviewable_topic_assets as find_cached_reviewable_topic_assets,\n)\n\nfrom .utils import (\n CreateTestUsers,\n create_asset,\n create_campaign,\n create_item,\n create_project,\n create_topic,\n create_transcription,\n)\n\n\nclass NextReviewableTopicAssetTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anon = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(sequence=1)\n self.asset2 = create_asset(\n item=self.asset1.item, sequence=2, slug=\"test-asset-2\"\n )\n self.topic = create_topic(project=self.asset1.item.project)\n\n def test_find_new_reviewable_topic_assets_filters_correctly(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n\n queryset = find_new_reviewable_topic_assets(self.topic, self.user)\n self.assertIn(self.asset1, queryset)\n\n def test_find_new_reviewable_topic_assets_without_user(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n\n queryset = find_new_reviewable_topic_assets(self.topic, None)\n self.assertIn(self.asset1, queryset)\n\n def test_find_reviewable_topic_asset_from_next_table(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n\n NextReviewableTopicAsset.objects.create(\n asset=self.asset1,\n topic=self.topic,\n item=self.asset1.item,\n item_item_id=self.asset1.item.item_id,\n project=self.asset1.item.project,\n project_slug=self.asset1.item.project.slug,\n sequence=self.asset1.sequence,\n transcriber_ids=[],\n )\n\n asset = find_reviewable_topic_asset(self.topic, self.user)\n self.assertEqual(asset, self.asset1)\n\n @patch(\"concordia.utils.next_asset.reviewable.topic.get_registered_task\")\n def test_find_reviewable_topic_asset_falls_back_and_spawns_task(\n self, mock_get_task\n ):\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset = find_reviewable_topic_asset(self.topic, self.user)\n self.assertEqual(asset, self.asset2)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.topic.get_registered_task\")\n def test_find_next_reviewable_topic_asset_orders_and_falls_back(\n self, mock_get_task\n ):\n \"\"\"\n With short-circuiting: project-level returns\n the eligible asset and we do not spawn a task.\n \"\"\"\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset = find_next_reviewable_topic_asset(\n self.topic,\n self.user,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(asset, self.asset1)\n # Short-circuit satisfied -> no cache fallback -> no task spawned\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.topic.get_registered_task\")\n def test_find_next_reviewable_topic_asset_when_next_asset_exists(\n self, mock_get_task\n ):\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n NextReviewableTopicAsset.objects.create(\n asset=self.asset2,\n topic=self.topic,\n item=self.asset2.item,\n item_item_id=self.asset2.item.item_id,\n project=self.asset2.item.project,\n project_slug=self.asset2.item.project.slug,\n sequence=self.asset2.sequence,\n transcriber_ids=[],\n )\n\n asset = find_next_reviewable_topic_asset(\n self.topic,\n self.user,\n project_slug=self.asset2.item.project.slug,\n item_id=self.asset2.item.item_id,\n original_asset_id=self.asset2.id - 1,\n )\n self.assertEqual(asset, self.asset2)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n def test_short_circuit_same_item_topic_excludes_users_own_work(self):\n mine = self.asset1\n other = self.asset2\n create_transcription(asset=mine, user=self.user, submitted=now())\n create_transcription(asset=other, user=self.anon, submitted=now())\n\n chosen = find_next_reviewable_topic_asset(\n self.topic,\n self.user,\n project_slug=mine.item.project.slug,\n item_id=mine.item.item_id,\n original_asset_id=mine.id,\n )\n self.assertEqual(chosen, other)\n\n def test_item_short_circuit_topic_reviewable_respects_after_and_reservations(self):\n asset3 = create_asset(item=self.asset1.item, sequence=3, slug=\"rev-topic-a3\")\n for asset in (self.asset1, self.asset2, asset3):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n\n chosen = find_next_reviewable_topic_asset(\n self.topic,\n self.user,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, self.asset2)\n\n AssetTranscriptionReservation.objects.create(\n asset=self.asset2, reservation_token=\"rv\" # nosec\n )\n chosen2 = find_next_reviewable_topic_asset(\n self.topic,\n self.user,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen2, asset3)\n\n @patch(\"concordia.utils.next_asset.reviewable.topic.get_registered_task\")\n def test_cache_excludes_user_and_triggers_spawn_task_topic(self, mock_get_task):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n create_transcription(asset=self.asset1, user=self.user, submitted=now())\n NextReviewableTopicAsset.objects.create(\n asset=self.asset1,\n topic=self.topic,\n item=self.asset1.item,\n item_item_id=self.asset1.item.item_id,\n project=self.asset1.item.project,\n project_slug=self.asset1.item.project.slug,\n sequence=self.asset1.sequence,\n transcriber_ids=[self.user.id],\n )\n\n other = create_asset(item=self.asset1.item, sequence=3, slug=\"rev-topic-other\")\n create_transcription(asset=other, user=self.anon, submitted=now())\n\n chosen = find_next_reviewable_topic_asset(\n self.topic,\n self.user,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=None,\n )\n self.assertEqual(chosen, other)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n def test_find_next_reviewable_topic_assets_excludes_user(self):\n create_transcription(asset=self.asset1, user=self.user, submitted=now())\n NextReviewableTopicAsset.objects.create(\n asset=self.asset1,\n topic=self.topic,\n item=self.asset1.item,\n item_item_id=self.asset1.item.item_id,\n project=self.asset1.item.project,\n project_slug=self.asset1.item.project.slug,\n sequence=self.asset1.sequence,\n transcriber_ids=[self.user.id],\n )\n NextReviewableTopicAsset.objects.create(\n asset=self.asset2,\n topic=self.topic,\n item=self.asset2.item,\n item_item_id=self.asset2.item.item_id,\n project=self.asset2.item.project,\n project_slug=self.asset2.item.project.slug,\n sequence=self.asset2.sequence,\n transcriber_ids=[],\n )\n queryset = find_cached_reviewable_topic_assets(self.topic, self.user)\n self.assertNotIn(self.asset1.id, queryset.values_list(\"asset_id\", flat=True))\n self.assertIn(self.asset2.id, queryset.values_list(\"asset_id\", flat=True))\n\n @patch(\"concordia.utils.next_asset.reviewable.topic.get_registered_task\")\n def test_next_reviewable_cached_path_when_short_circuits_fail_topic(\n self, mock_get_task\n ):\n \"\"\"\n Item+project short-circuits fail (only user's work), so we should pull\n from the cached table and not spawn a task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Only user's submitted work in current item/project\n create_transcription(asset=self.asset1, user=self.user, submitted=now())\n create_transcription(asset=self.asset2, user=self.user, submitted=now())\n\n # Cached eligible asset in another project (not reachable via short-circuit)\n cached_project = create_project(\n campaign=self.asset1.campaign,\n slug=\"topic-cached-proj\",\n title=\"topic-cached-proj\",\n )\n cached_item = create_item(project=cached_project, item_id=\"topic-cached-item\")\n cached_asset = create_asset(item=cached_item, slug=\"topic-cached-asset\")\n create_transcription(asset=cached_asset, user=self.anon, submitted=now())\n\n NextReviewableTopicAsset.objects.create(\n asset=cached_asset,\n topic=self.topic,\n item=cached_asset.item,\n item_item_id=cached_asset.item.item_id,\n project=cached_asset.item.project,\n project_slug=cached_asset.item.project.slug,\n sequence=cached_asset.sequence,\n transcriber_ids=[],\n )\n\n chosen = find_next_reviewable_topic_asset(\n self.topic,\n self.user,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, cached_asset)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.topic.get_registered_task\")\n def test_next_reviewable_uses_cache_when_bypassing_short_circuits_topic(\n self, mock_get_task\n ):\n \"\"\"\n Pass blanks for project/item so we bypass short-circuits and hit cache.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n cached_project = create_project(\n campaign=self.asset1.campaign,\n slug=\"topic-cached-proj-2\",\n title=\"topic-cached-proj-2\",\n )\n cached_item = create_item(project=cached_project, item_id=\"topic-cached-item-2\")\n cached_asset = create_asset(item=cached_item, slug=\"topic-cached-asset-2\")\n create_transcription(asset=cached_asset, user=self.anon, submitted=now())\n\n NextReviewableTopicAsset.objects.create(\n asset=cached_asset,\n topic=self.topic,\n item=cached_asset.item,\n item_item_id=cached_asset.item.item_id,\n project=cached_asset.item.project,\n project_slug=cached_asset.item.project.slug,\n sequence=cached_asset.sequence,\n transcriber_ids=[],\n )\n\n chosen = find_next_reviewable_topic_asset(\n self.topic,\n self.user,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=None,\n )\n self.assertEqual(chosen, cached_asset)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.topic.get_registered_task\")\n def test_next_reviewable_manual_fallback_no_after_spawns_and_picks_lowest_seq_topic(\n self, mock_get_task\n ):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n other_item = create_item(project=self.asset1.item.project, item_id=\"t-mf-item\")\n asset_x = create_asset(item=other_item, sequence=7, slug=\"t-mf-x\")\n asset_y = create_asset(item=other_item, sequence=8, slug=\"t-mf-y\")\n create_transcription(asset=asset_x, user=self.anon, submitted=now())\n create_transcription(asset=asset_y, user=self.anon, submitted=now())\n\n chosen = find_next_reviewable_topic_asset(\n self.topic,\n self.user,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=None,\n )\n self.assertEqual(chosen, asset_x)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.reviewable.topic.get_registered_task\")\n def test_next_reviewable_manual_fallback_invalid_after_str_topic(\n self, mock_get_task\n ):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"t-mf-item-2\"\n )\n asset_a = create_asset(item=other_item, sequence=1, slug=\"t-mf-a\")\n asset_b = create_asset(item=other_item, sequence=2, slug=\"t-mf-b\")\n create_transcription(asset=asset_a, user=self.anon, submitted=now())\n create_transcription(asset=asset_b, user=self.anon, submitted=now())\n\n chosen = find_next_reviewable_topic_asset(\n self.topic,\n self.user,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=\"not-an-int\",\n )\n self.assertEqual(chosen, asset_a)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n\nclass ReviewableTopicInternalsTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anon = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(sequence=1, slug=\"rt-a1\")\n self.asset2 = create_asset(item=self.asset1.item, sequence=2, slug=\"rt-a2\")\n self.topic = create_topic(project=self.asset1.item.project)\n\n def test_topic_reserved_asset_ids_subq_unfiltered(self):\n # Reservation tied to this test topic\n AssetTranscriptionReservation.objects.create(\n asset=self.asset1,\n reservation_token=\"rt-r1\", # nosec\n )\n # Reservation in entirely different campaign/project\n other_campaign = create_campaign(slug=\"rt-camp-x\", title=\"rt-camp-x\")\n other_project = create_project(\n campaign=other_campaign, slug=\"rt-proj-x\", title=\"rt-proj-x\"\n )\n other_item = create_item(project=other_project, item_id=\"rt-item-x\")\n other_asset = create_asset(item=other_item, slug=\"rt-asset-x\")\n AssetTranscriptionReservation.objects.create(\n asset=other_asset,\n reservation_token=\"rt-r2\", # nosec\n )\n\n id_set = set(topic_reserved_asset_ids_subq().values_list(\"asset_id\", flat=True))\n self.assertIn(self.asset1.id, id_set)\n self.assertIn(other_asset.id, id_set)\n\n def test_topic_eligible_reviewable_base_qs_excludes_user_and_requires_submitted(\n self,\n ):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.user, submitted=now())\n asset3 = create_asset(item=self.asset1.item, sequence=3, slug=\"rt-a3\")\n # asset3 has no submitted timestamp -> not SUBMITTED\n\n queryset_user = topic_eligible_reviewable_base_qs(self.topic, user=self.user)\n self.assertIn(self.asset1, queryset_user)\n self.assertNotIn(self.asset2, queryset_user)\n self.assertNotIn(asset3, queryset_user)\n\n queryset_none = topic_eligible_reviewable_base_qs(self.topic, user=None)\n self.assertIn(self.asset1, queryset_none)\n self.assertIn(self.asset2, queryset_none)\n self.assertNotIn(asset3, queryset_none)\n\n def test_topic_next_seq_after_none_missing_and_valid(self):\n self.assertIsNone(topic_next_seq_after(None))\n self.assertIsNone(topic_next_seq_after(987654321))\n self.assertEqual(topic_next_seq_after(self.asset2.pk), self.asset2.sequence)\n\n def test_topic_find_reviewable_in_item_after_none_returns_first(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n chosen = topic_find_reviewable_in_item(\n self.topic,\n self.user,\n item_id=self.asset1.item.item_id,\n after_asset_pk=None,\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_topic_find_reviewable_in_item_after_asset_in_other_item_ignores_gate(\n self,\n ):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"rt-other-item\"\n )\n other_asset = create_asset(item=other_item, slug=\"rt-other-asset\")\n create_transcription(asset=other_asset, user=self.anon, submitted=now())\n\n chosen = topic_find_reviewable_in_item(\n self.topic,\n self.user,\n item_id=self.asset1.item.item_id,\n after_asset_pk=other_asset.id,\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_topic_find_reviewable_in_item_after_asset_missing_ignores_gate(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n chosen = topic_find_reviewable_in_item(\n self.topic,\n self.user,\n item_id=self.asset1.item.item_id,\n after_asset_pk=123456789,\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_topic_find_reviewable_in_item_after_asset_sidc_ignores_gate(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n other_campaign = create_campaign(slug=\"rt-camp-b\", title=\"rt-camp-b\")\n other_project = create_project(\n campaign=other_campaign, slug=\"rt-proj-b\", title=\"rt-proj-b\"\n )\n other_item = create_item(\n project=other_project, item_id=self.asset1.item.item_id\n )\n cross_asset = create_asset(item=other_item, slug=\"rt-cross\")\n create_transcription(asset=cross_asset, user=self.anon, submitted=now())\n\n chosen = topic_find_reviewable_in_item(\n self.topic,\n self.user,\n item_id=self.asset1.item.item_id,\n after_asset_pk=cross_asset.id,\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_topic_find_reviewable_in_project_orders_and_excludes_user(self):\n project = self.asset1.item.project\n other_item = create_item(project=project, item_id=\"rt-p-item\")\n mine = create_asset(item=other_item, sequence=1, slug=\"rt-p-mine\")\n theirs = create_asset(item=other_item, sequence=2, slug=\"rt-p-theirs\")\n create_transcription(asset=mine, user=self.user, submitted=now())\n create_transcription(asset=theirs, user=self.anon, submitted=now())\n\n chosen = topic_find_reviewable_in_project(\n self.topic,\n self.user,\n project_slug=project.slug,\n after_asset_pk=self.asset1.id,\n )\n self.assertEqual(chosen, theirs)\n\n def test_topic_find_reviewable_in_project_returns_none_when_only_users_work(self):\n project = self.asset1.item.project\n other_item = create_item(project=project, item_id=\"rt-p2\")\n mine = create_asset(item=other_item, sequence=1, slug=\"rt-p2-mine\")\n create_transcription(asset=mine, user=self.user, submitted=now())\n\n chosen = topic_find_reviewable_in_project(\n self.topic,\n self.user,\n project_slug=project.slug,\n after_asset_pk=self.asset1.id,\n )\n self.assertIsNone(chosen)\n\n def test_find_and_order_potential_reviewable_topic_assets_ordering(self):\n base_item = self.asset1.item\n\n same_item_next = create_asset(item=base_item, sequence=10, slug=\"rt-ci-next\")\n other_item_same_project = create_asset(\n item=create_item(project=base_item.project, item_id=\"rt-it-2\"),\n sequence=5,\n slug=\"rt-p-next\",\n )\n other_project = create_project(\n campaign=self.asset1.campaign, slug=\"rt-proj\", title=\"rt-proj\"\n )\n other_project_item = create_item(project=other_project, item_id=\"rt-it-3\")\n other_project_asset = create_asset(\n item=other_project_item, sequence=1, slug=\"rt-op\"\n )\n\n for asset in (same_item_next, other_item_same_project, other_project_asset):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n NextReviewableTopicAsset.objects.create(\n asset=asset,\n topic=self.topic,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n sequence=asset.sequence,\n transcriber_ids=[],\n )\n\n ordered = find_and_order_potential_reviewable_topic_assets(\n self.topic,\n self.user,\n project_slug=base_item.project.slug,\n item_id=base_item.item_id,\n asset_pk=self.asset1.id,\n ).values_list(\"asset_id\", flat=True)\n\n ordered = list(ordered)\n self.assertEqual(ordered[0], same_item_next.id)\n self.assertEqual(ordered[1], other_item_same_project.id)\n self.assertIn(other_project_asset.id, ordered[2:])\n\n def test_order_potential_without_after_prefers_item_then_project_topic(self):\n base_item = self.asset1.item\n\n same_item = create_asset(item=base_item, sequence=9, slug=\"rt-ci-none\")\n same_project = create_asset(\n item=create_item(project=base_item.project, item_id=\"rt-it-np\"),\n sequence=2,\n slug=\"rt-p-none\",\n )\n other_project = create_project(\n campaign=self.asset1.campaign, slug=\"rt-proj-none\", title=\"rt-proj-none\"\n )\n other_project_item = create_item(project=other_project, item_id=\"rt-it-op-none\")\n other_project_asset = create_asset(\n item=other_project_item, sequence=1, slug=\"rt-op-none\"\n )\n for asset in (same_item, same_project, other_project_asset):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n NextReviewableTopicAsset.objects.create(\n asset=asset,\n topic=self.topic,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n sequence=asset.sequence,\n transcriber_ids=[],\n )\n\n ordered = find_and_order_potential_reviewable_topic_assets(\n self.topic,\n self.user,\n project_slug=base_item.project.slug,\n item_id=base_item.item_id,\n asset_pk=None, # next_asset==0 branch\n ).values_list(\"asset_id\", flat=True)\n\n ordered = list(ordered)\n self.assertEqual(ordered[0], same_item.id)\n self.assertEqual(ordered[1], same_project.id)\n self.assertIn(other_project_asset.id, ordered[2:])\n\n def test_find_invalid_next_reviewable_topic_assets_reserved_and_wrong_status(\n self,\n ):\n # Reserved\n reserved_asset = create_asset(\n item=self.asset1.item, sequence=30, slug=\"rt-inv-res\"\n )\n create_transcription(asset=reserved_asset, user=self.anon, submitted=now())\n AssetTranscriptionReservation.objects.create(\n asset=reserved_asset, reservation_token=\"rt-rv\" # nosec\n )\n NextReviewableTopicAsset.objects.create(\n asset=reserved_asset,\n topic=self.topic,\n item=reserved_asset.item,\n item_item_id=reserved_asset.item.item_id,\n project=reserved_asset.item.project,\n project_slug=reserved_asset.item.project.slug,\n sequence=reserved_asset.sequence,\n transcriber_ids=[],\n )\n\n # Wrong status (IN_PROGRESS)\n wrong_status_asset = create_asset(\n item=self.asset1.item, sequence=31, slug=\"rt-inv-wrong\"\n )\n create_transcription(asset=wrong_status_asset, user=self.anon)\n NextReviewableTopicAsset.objects.create(\n asset=wrong_status_asset,\n topic=self.topic,\n item=wrong_status_asset.item,\n item_item_id=wrong_status_asset.item.item_id,\n project=wrong_status_asset.item.project,\n project_slug=wrong_status_asset.item.project.slug,\n sequence=wrong_status_asset.sequence,\n transcriber_ids=[],\n )\n\n invalid_ids = list(\n find_invalid_next_reviewable_topic_assets(self.topic.id).values_list(\n \"asset_id\", flat=True\n )\n )\n self.assertIn(reserved_asset.id, invalid_ids)\n self.assertIn(wrong_status_asset.id, invalid_ids)\n\n def test_topic_project_short_circuit_internal_skips_reserved_first(self):\n project = self.asset1.item.project\n item2 = create_item(project=project, item_id=\"rt-proj-int\")\n first = create_asset(item=item2, sequence=1, slug=\"rt-proj-int-1\")\n second = create_asset(item=item2, sequence=2, slug=\"rt-proj-int-2\")\n for asset in (first, second):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n AssetTranscriptionReservation.objects.create(\n asset=first, reservation_token=\"rt-proj-int\" # nosec\n )\n\n chosen = topic_find_reviewable_in_project(\n self.topic,\n self.user,\n project_slug=project.slug,\n after_asset_pk=self.asset1.id,\n )\n self.assertEqual(chosen, second)\n\n def test_topic_item_short_circuit_internal_excludes_users_own_work(self):\n mine = self.asset1\n other = self.asset2\n create_transcription(asset=mine, user=self.user, submitted=now())\n create_transcription(asset=other, user=self.anon, submitted=now())\n\n chosen = topic_find_reviewable_in_item(\n self.topic,\n self.user,\n item_id=mine.item.item_id,\n after_asset_pk=None,\n )\n self.assertEqual(chosen, other)\n\n def test_topic_item_short_circuit_internal_applies_after_and_skips_reserved(self):\n asset3 = create_asset(item=self.asset1.item, sequence=3, slug=\"rt-int-a3\")\n for asset in (self.asset1, self.asset2, asset3):\n create_transcription(asset=asset, user=self.anon, submitted=now())\n AssetTranscriptionReservation.objects.create(\n asset=self.asset2, reservation_token=\"rt-int-rv\" # nosec\n )\n\n chosen = topic_find_reviewable_in_item(\n self.topic,\n self.user,\n item_id=self.asset1.item.item_id,\n after_asset_pk=self.asset1.id,\n )\n self.assertEqual(chosen, asset3)\n"
},
{
"path": "concordia/tests/test_utils_next_asset_transcribable_campaign.py",
"content": "from unittest.mock import MagicMock, patch\n\nfrom django.test import TestCase\nfrom django.utils.timezone import now\n\nfrom concordia.models import (\n Asset,\n AssetTranscriptionReservation,\n NextTranscribableCampaignAsset,\n TranscriptionStatus,\n)\nfrom concordia.utils import get_anonymous_user\nfrom concordia.utils.next_asset import (\n find_new_transcribable_campaign_assets,\n find_next_transcribable_campaign_asset,\n find_transcribable_campaign_asset,\n)\nfrom concordia.utils.next_asset.transcribable.campaign import (\n _eligible_transcribable_base_qs as tc_eligible_base_qs,\n)\nfrom concordia.utils.next_asset.transcribable.campaign import (\n _find_transcribable_in_item as tc_find_in_item,\n)\nfrom concordia.utils.next_asset.transcribable.campaign import (\n _find_transcribable_not_started_in_project as tc_find_ns_in_proj,\n)\nfrom concordia.utils.next_asset.transcribable.campaign import (\n _next_seq_after as tc_next_seq_after,\n)\nfrom concordia.utils.next_asset.transcribable.campaign import (\n _order_unstarted_first as tc_order_unstarted_first,\n)\nfrom concordia.utils.next_asset.transcribable.campaign import (\n _reserved_asset_ids_subq as tc_reserved_ids_subq,\n)\nfrom concordia.utils.next_asset.transcribable.campaign import (\n find_and_order_potential_transcribable_campaign_assets,\n find_invalid_next_transcribable_campaign_assets,\n)\nfrom concordia.utils.next_asset.transcribable.campaign import (\n find_next_transcribable_campaign_assets as find_cached_transcribable_assets,\n)\n\nfrom .utils import (\n CreateTestUsers,\n create_asset,\n create_campaign,\n create_item,\n create_project,\n create_transcription,\n)\n\n\nclass NextTranscribableCampaignAssetTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anon = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(\n sequence=1, slug=\"test-asset-1\", title=\"Test Asset 1\"\n )\n self.asset2 = create_asset(\n item=self.asset1.item, sequence=2, slug=\"test-asset-2\", title=\"Test Asset 2\"\n )\n self.campaign = self.asset1.campaign\n\n def test_find_new_transcribable_campaign_assets_filters_correctly(self):\n create_transcription(\n asset=self.asset1,\n user=self.anon,\n submitted=now(),\n )\n\n queryset = find_new_transcribable_campaign_assets(self.campaign)\n self.assertNotIn(self.asset1, queryset)\n self.assertIn(self.asset2, queryset)\n\n def test_find_transcribable_campaign_asset_from_next_table(self):\n NextTranscribableCampaignAsset.objects.create(\n asset=self.asset1,\n campaign=self.campaign,\n item=self.asset1.item,\n item_item_id=self.asset1.item.item_id,\n project=self.asset1.item.project,\n project_slug=self.asset1.item.project.slug,\n sequence=self.asset1.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n\n asset = find_transcribable_campaign_asset(self.campaign)\n self.assertEqual(asset, self.asset1)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_find_transcribable_campaign_asset_falls_back_and_spawns_task(\n self, mock_get_task\n ):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset = find_transcribable_campaign_asset(self.campaign)\n self.assertEqual(asset, self.asset1)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_find_next_transcribable_campaign_asset_orders_and_falls_back(\n self, mock_get_task\n ):\n \"\"\"\n With short-circuiting: item-level returns the next asset\n and we do not spawn a task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(asset, self.asset2)\n # Short-circuit satisfied -> no cache fallback -> no task spawned\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_find_next_transcribable_campaign_asset_when_next_asset_exists(\n self, mock_get_task\n ):\n # Make asset2 eligible (IN_PROGRESS)\n create_transcription(\n asset=self.asset2,\n user=self.anon,\n )\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Cache has asset2\n NextTranscribableCampaignAsset.objects.create(\n asset=self.asset2,\n campaign=self.campaign,\n item=self.asset2.item,\n item_item_id=self.asset2.item.item_id,\n project=self.asset2.item.project,\n project_slug=self.asset2.item.project.slug,\n sequence=self.asset2.sequence,\n transcription_status=TranscriptionStatus.IN_PROGRESS,\n )\n\n # Bypass item/project short-circuits so we hit the cache\n asset = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=None,\n )\n self.assertEqual(asset, self.asset2)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n def test_short_circuit_same_item_respects_after_sequence_and_reservations(self):\n \"\"\"\n Same item short-circuit:\n - Picks the next by sequence (> original)\n - Skips reserved assets\n \"\"\"\n asset3 = create_asset(\n item=self.asset1.item, sequence=3, slug=\"test-asset-3\", title=\"Test Asset 3\"\n )\n\n # Normal: after asset1 => choose asset2\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, self.asset2)\n\n # Reserve asset2 => should skip to asset3\n AssetTranscriptionReservation.objects.create(\n asset=self.asset2, reservation_token=\"tkn\" # nosec\n )\n chosen2 = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen2, asset3)\n\n def test_project_short_circuit_prefers_not_started_over_in_progress(self):\n \"\"\"\n When item-level has no eligible assets, project-level should:\n - Prefer NOT_STARTED over IN_PROGRESS\n - Order by (item_id, sequence) within same status\n \"\"\"\n # Exhaust item: mark both item assets submitted\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"proj-only-item\"\n )\n in_progress_asset = create_asset(\n item=other_item, slug=\"proj-inprog\", title=\"Proj InProg\"\n )\n create_transcription(asset=in_progress_asset, user=self.anon) # IN_PROGRESS\n\n not_started_asset = create_asset(\n item=other_item, slug=\"proj-notstarted\", title=\"Proj NotStarted\"\n )\n\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id, # same item, but it's exhausted\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, not_started_asset)\n\n def test_project_short_circuit_when_item_id_empty_string(self):\n \"\"\"\n If item_id is '', skip item short-circuit and use project-level.\n \"\"\"\n other_item = create_item(project=self.asset1.item.project, item_id=\"proj2\")\n project_asset = create_asset(\n item=other_item, slug=\"proj-asset\", title=\"Proj Asset\"\n )\n # Make current item ineligible\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=\"\", # empty skips item short-circuit\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, project_asset)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_find_transcribable_campaign_asset_none_spawns(self, mock_get_task):\n \"\"\"\n When no NOT_STARTED/IN_PROGRESS exist, return None and trigger populate.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n # Make both assets SUBMITTED (no transcribable remain)\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n asset = find_transcribable_campaign_asset(self.campaign)\n self.assertIsNone(asset)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_next_transcribable_manual_no_after_prefers_not_started(\n self, mock_get_task\n ):\n \"\"\"\n With no short-circuit and empty cache, pick NOT_STARTED and spawn task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n campaign2 = create_campaign(slug=\"tc-na-camp\", title=\"tc-na-camp\")\n project2 = create_project(\n campaign=campaign2, slug=\"tc-na-proj\", title=\"tc-na-proj\"\n )\n item2 = create_item(project=project2, item_id=\"tc-na-item\")\n\n not_started_asset = create_asset(\n item=item2, sequence=2, slug=\"tc-na-ns\", title=\"TC NA NS\"\n )\n in_progress_asset = create_asset(\n item=item2, sequence=1, slug=\"tc-na-ip\", title=\"TC NA IP\"\n )\n create_transcription(asset=in_progress_asset, user=self.anon) # IN_PROGRESS\n\n chosen = find_next_transcribable_campaign_asset(\n campaign2, project_slug=\"\", item_id=\"\", original_asset_id=None\n )\n self.assertEqual(chosen, not_started_asset)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_next_transcribable_manual_invalid_after_str(self, mock_get_task):\n \"\"\"\n Treat a non-integer \"after\" like None: choose NOT_STARTED and spawn task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Make existing setup assets ineligible for selection.\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"tc-mf-item-2\"\n )\n asset_a = create_asset(\n item=other_item, sequence=1, slug=\"tc-mf-a\", title=\"TC MF A\"\n )\n asset_b = create_asset(\n item=other_item, sequence=2, slug=\"tc-mf-b\", title=\"TC MF B\"\n )\n create_transcription(asset=asset_b, user=self.anon) # IN_PROGRESS\n\n chosen = find_next_transcribable_campaign_asset(\n self.campaign, project_slug=\"\", item_id=\"\", original_asset_id=None\n )\n self.assertEqual(chosen, asset_a)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_next_transcribable_none_anywhere_spawns(self, mock_get_task):\n \"\"\"\n With no cache and no manual candidates: return None; do not spawn a task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n campaign2 = create_campaign(slug=\"tc-none-camp\", title=\"tc-none-camp\")\n\n chosen = find_next_transcribable_campaign_asset(\n campaign2, project_slug=\"\", item_id=\"\", original_asset_id=None\n )\n self.assertIsNone(chosen)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n def test_item_short_circuit_missing_after_pk_treated_as_none_top(self):\n # Both assets are NOT_STARTED in the same item. Give a missing \"after\".\n missing_pk = 987_654_321\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=missing_pk,\n )\n # With no valid \"after\" seq, returns the first NOT_STARTED (asset1).\n self.assertEqual(chosen, self.asset1)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_cache_excludes_original_pk_and_chooses_next(self, mock_get_task):\n # Two cached rows; original points at the first -> second should be chosen.\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"tc-cache-pk-item\"\n )\n first = create_asset(item=other_item, sequence=1, slug=\"tc-cache-first\")\n second = create_asset(item=other_item, sequence=2, slug=\"tc-cache-second\")\n\n for asset in (first, second):\n NextTranscribableCampaignAsset.objects.create(\n asset=asset,\n campaign=self.campaign,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n sequence=asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=first.id,\n )\n self.assertEqual(chosen, second)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n def test_project_short_circuit_without_original_id(self):\n # Exhaust current item; ensure project-level returns NOT_STARTED with\n # original_asset_id=None.\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"tc-proj-no-orig\"\n )\n pick = create_asset(item=other_item, sequence=5, slug=\"tc-proj-pick\")\n\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=None,\n )\n self.assertEqual(chosen, pick)\n\n def test_item_short_circuit_after_pk_in_other_item_ignores_gate(self):\n # Original PK exists but belongs to a different item; treat as no \"after\".\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"tc-oth-it-ignores-gate\"\n )\n other_asset = create_asset(item=other_item, slug=\"tc-oth-a-ignores-gate\")\n\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=other_asset.id,\n )\n # With no valid \"after\" in this item, pick first NOT_STARTED by sequence.\n self.assertEqual(chosen, self.asset1)\n\n def test_next_transcribable_after_pk_missing_treats_as_no_after(self):\n \"\"\"\n Missing original_asset_id -> ignore 'after' gate and pick first NS in item.\n \"\"\"\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=987654321, # missing\n )\n self.assertEqual(chosen, self.asset1)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_no_ns_anywhere_and_no_ip_in_item_returns_none(self, mock_get_task):\n \"\"\"\n With item_id present: no NOT_STARTED anywhere and no same-item IN_PROGRESS\n so return None and do not spawn a task.\n \"\"\"\n # Exhaust the only item in the project/campaign.\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n got = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertIsNone(got)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n def test_after_pk_digit_string_missing_treats_as_no_after(self):\n \"\"\"\n original_asset_id is a digit string for a missing PK -> treat like no 'after'.\n Covers the DoesNotExist branch distinct from non-digit ValueError.\n \"\"\"\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=\"987654321\", # digit string, no such Asset\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_same_item_inprogress_selected_when_no_ns_and_no_after(self):\n \"\"\"\n With item_id present, no NOT_STARTED anywhere and original_asset_id=None,\n pick same-item IN_PROGRESS (after_seq is None path).\n \"\"\"\n create_transcription(asset=self.asset2, user=self.anon) # IN_PROGRESS\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n\n got = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=None, # after_seq is None in IP fallback\n )\n self.assertEqual(got, self.asset2)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_manual_invalid_after_str_campaign_valueerror_branch(self, mock_get_task):\n \"\"\"\n original_asset_id is a non-digit string -> ValueError path.\n Bypass short-circuits and empty cache => manual picks first NOT_STARTED\n and spawns populate task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=\"not-an-int\",\n )\n self.assertEqual(chosen, self.asset1) # first NOT_STARTED by ordering\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n\nclass TranscribableCampaignInternalsTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anon = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(sequence=1, slug=\"tc-a1\")\n self.asset2 = create_asset(item=self.asset1.item, sequence=2, slug=\"tc-a2\")\n self.campaign = self.asset1.campaign\n\n def test_new_transcribable_excludes_reserved_and_cached(self):\n reserved_asset = create_asset(item=self.asset1.item, sequence=3, slug=\"tc-res\")\n cached_asset = create_asset(item=self.asset1.item, sequence=4, slug=\"tc-cached\")\n # Make both potentially transcribable\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n # Reserve one and cache the other\n AssetTranscriptionReservation.objects.create(\n asset=reserved_asset, reservation_token=\"tc-rv\" # nosec\n )\n NextTranscribableCampaignAsset.objects.create(\n asset=cached_asset,\n campaign=self.campaign,\n item=cached_asset.item,\n item_item_id=cached_asset.item.item_id,\n project=cached_asset.item.project,\n project_slug=cached_asset.item.project.slug,\n sequence=cached_asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n queryset = find_new_transcribable_campaign_assets(self.campaign)\n self.assertNotIn(reserved_asset, queryset)\n self.assertNotIn(cached_asset, queryset)\n\n def test_order_potential_transcribable_pref(self):\n \"\"\"\n Cached ordering should favor next id, then same project, then same item.\n \"\"\"\n base_item = self.asset1.item\n same_item_next = create_asset(item=base_item, sequence=10, slug=\"tc-ci-next\")\n same_project = create_asset(\n item=create_item(project=base_item.project, item_id=\"tc-it-2\"),\n sequence=5,\n slug=\"tc-p-next\",\n )\n other_project_asset = create_asset(\n item=create_item(\n project=create_project(\n campaign=self.campaign, slug=\"tc-op-proj\", title=\"tc-op-proj\"\n ),\n item_id=\"tc-op-item\",\n ),\n sequence=1,\n slug=\"tc-op\",\n )\n for asset in (same_item_next, same_project, other_project_asset):\n NextTranscribableCampaignAsset.objects.create(\n asset=asset,\n campaign=self.campaign,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n sequence=asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n ordered = find_and_order_potential_transcribable_campaign_assets(\n self.campaign,\n project_slug=base_item.project.slug,\n item_id=base_item.item_id,\n asset_pk=self.asset1.id,\n ).values_list(\"asset_id\", flat=True)\n ordered = list(ordered)\n self.assertEqual(ordered[0], same_item_next.id)\n self.assertEqual(ordered[1], same_project.id)\n self.assertIn(other_project_asset.id, ordered[2:])\n\n def test_order_potential_transcribable_no_after(self):\n \"\"\"\n With no 'after', prefer same item, then same project.\n \"\"\"\n base_item = self.asset1.item\n same_item = create_asset(item=base_item, sequence=9, slug=\"tc-ci-none\")\n same_project = create_asset(\n item=create_item(project=base_item.project, item_id=\"tc-it-np\"),\n sequence=2,\n slug=\"tc-p-none\",\n )\n other_project_asset = create_asset(\n item=create_item(\n project=create_project(\n campaign=self.campaign, slug=\"tc-proj-none\", title=\"tc-proj-none\"\n ),\n item_id=\"tc-it-op-none\",\n ),\n sequence=1,\n slug=\"tc-op-none\",\n )\n for asset in (same_item, same_project, other_project_asset):\n NextTranscribableCampaignAsset.objects.create(\n asset=asset,\n campaign=self.campaign,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n sequence=asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n ordered = find_and_order_potential_transcribable_campaign_assets(\n self.campaign,\n project_slug=base_item.project.slug,\n item_id=base_item.item_id,\n asset_pk=None,\n ).values_list(\"asset_id\", flat=True)\n ordered = list(ordered)\n self.assertEqual(ordered[0], same_item.id)\n self.assertEqual(ordered[1], same_project.id)\n self.assertIn(other_project_asset.id, ordered[2:])\n\n def test_invalid_next_transcribable_reserved_and_submitted(self):\n \"\"\"\n Invalid cache rows include reserved or SUBMITTED assets.\n \"\"\"\n reserved_asset = create_asset(\n item=self.asset1.item, sequence=30, slug=\"tc-inv-res\"\n )\n AssetTranscriptionReservation.objects.create(\n asset=reserved_asset, reservation_token=\"tc-rv-2\" # nosec\n )\n NextTranscribableCampaignAsset.objects.create(\n asset=reserved_asset,\n campaign=self.campaign,\n item=reserved_asset.item,\n item_item_id=reserved_asset.item.item_id,\n project=reserved_asset.item.project,\n project_slug=reserved_asset.item.project.slug,\n sequence=reserved_asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n wrong_status_asset = create_asset(\n item=self.asset1.item, sequence=31, slug=\"tc-inv-wrong\"\n )\n create_transcription(asset=wrong_status_asset, user=self.anon, submitted=now())\n NextTranscribableCampaignAsset.objects.create(\n asset=wrong_status_asset,\n campaign=self.campaign,\n item=wrong_status_asset.item,\n item_item_id=wrong_status_asset.item.item_id,\n project=wrong_status_asset.item.project,\n project_slug=wrong_status_asset.item.project.slug,\n sequence=wrong_status_asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n bad = list(\n find_invalid_next_transcribable_campaign_assets(\n self.campaign.id\n ).values_list(\"asset_id\", flat=True)\n )\n self.assertIn(reserved_asset.id, bad)\n self.assertIn(wrong_status_asset.id, bad)\n\n\nclass TranscribableCampaignMoreInternalsTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anon = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(sequence=1, slug=\"tc-more-a1\", title=\"TC More A1\")\n self.asset2 = create_asset(\n item=self.asset1.item, sequence=2, slug=\"tc-more-a2\", title=\"TC More A2\"\n )\n self.campaign = self.asset1.campaign\n\n def test_tc_reserved_ids_filters_to_campaign(self):\n AssetTranscriptionReservation.objects.create(\n asset=self.asset1, reservation_token=\"tc-res-here\" # nosec\n )\n other_campaign = create_campaign(slug=\"tc-other-c\", title=\"tc-other-c\")\n other_project = create_project(\n campaign=other_campaign, slug=\"tc-other-p\", title=\"tc-other-p\"\n )\n other_item = create_item(project=other_project, item_id=\"tc-other-it\")\n other_asset = create_asset(item=other_item, slug=\"tc-other-a\")\n AssetTranscriptionReservation.objects.create(\n asset=other_asset, reservation_token=\"tc-res-there\" # nosec\n )\n\n id_set = set(\n tc_reserved_ids_subq(self.campaign).values_list(\"asset_id\", flat=True)\n )\n self.assertIn(self.asset1.id, id_set)\n self.assertNotIn(other_asset.id, id_set)\n\n def test_tc_next_seq_after_variants(self):\n self.assertIsNone(tc_next_seq_after(None))\n self.assertIsNone(tc_next_seq_after(999_999_999))\n self.assertEqual(tc_next_seq_after(self.asset2.id), self.asset2.sequence)\n\n def test_tc_order_unstarted_first_prefers_not_started(self):\n create_transcription(asset=self.asset2, user=self.anon)\n queryset = Asset.objects.filter(id__in=[self.asset1.id, self.asset2.id])\n ordered = list(tc_order_unstarted_first(queryset).values_list(\"id\", flat=True))\n self.assertEqual(ordered[0], self.asset1.id)\n self.assertEqual(ordered[1], self.asset2.id)\n\n def test_find_in_item_after_none_returns_first_not_started(self):\n item_id = self.asset1.item.item_id\n chosen = tc_find_in_item(self.campaign, item_id=item_id, after_asset_pk=None)\n self.assertEqual(chosen, self.asset1)\n\n def test_find_in_item_skips_inprog_and_reserved_and_advances(self):\n create_transcription(asset=self.asset1, user=self.anon)\n asset3 = create_asset(item=self.asset1.item, sequence=3, slug=\"tc-more-a3\")\n AssetTranscriptionReservation.objects.create(\n asset=asset3, reservation_token=\"tc-res-a3\" # nosec\n )\n chosen = tc_find_in_item(\n self.campaign,\n item_id=self.asset1.item.item_id,\n after_asset_pk=self.asset1.id,\n )\n self.assertEqual(chosen, self.asset2)\n\n def test_find_in_item_after_missing_excludes_id_only(self):\n missing_pk = 987654321\n chosen = tc_find_in_item(\n self.campaign, item_id=self.asset1.item.item_id, after_asset_pk=missing_pk\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_find_ns_in_proj_excludes_item_and_reserved(self):\n project = self.asset1.item.project\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n item2 = create_item(project=project, item_id=\"tc-more-it-2\")\n not_started1 = create_asset(item=item2, sequence=1, slug=\"tc-more-ns1\")\n not_started2 = create_asset(item=item2, sequence=2, slug=\"tc-more-ns2\")\n AssetTranscriptionReservation.objects.create(\n asset=not_started1, reservation_token=\"tc-res-ns1\" # nosec\n )\n chosen = tc_find_ns_in_proj(\n self.campaign,\n project_slug=project.slug,\n exclude_item_id=self.asset1.item.item_id,\n )\n self.assertEqual(chosen, not_started2)\n\n def test_find_ns_in_proj_blank_slug_none(self):\n self.assertIsNone(tc_find_ns_in_proj(self.campaign, project_slug=\"\"))\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_cache_same_item_is_ignored_then_manual_selects(self, mock_get_task):\n \"\"\"\n Same-item cache entries should be ignored; manual should return other item.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n NextTranscribableCampaignAsset.objects.create(\n asset=self.asset2,\n campaign=self.campaign,\n item=self.asset2.item,\n item_item_id=self.asset2.item.item_id,\n project=self.asset2.item.project,\n project_slug=self.asset2.item.project.slug,\n sequence=self.asset2.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n\n item2 = create_item(project=self.asset1.item.project, item_id=\"tc-more-it-man\")\n picked_asset = create_asset(item=item2, sequence=10, slug=\"tc-more-pick\")\n\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=\"\", # skip project-level short-circuit\n item_id=self.asset1.item.item_id, # forces same-item short-circuit first\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, picked_asset)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.campaign.get_registered_task\")\n def test_manual_excludes_original_pk_and_same_item(self, mock_get_task):\n \"\"\"\n Manual ranking must exclude the original asset and the current item.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n item2 = create_item(project=self.asset1.item.project, item_id=\"tc-more-it-3\")\n keep = create_asset(item=item2, sequence=1, slug=\"tc-more-keep\")\n toss = create_asset(item=item2, sequence=2, slug=\"tc-more-toss\")\n\n chosen = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=\"\",\n item_id=self.asset1.item.item_id,\n original_asset_id=toss.id,\n )\n self.assertEqual(chosen, keep)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n def test_same_item_inprog_after_when_no_not_started(self):\n \"\"\"\n If no NOT_STARTED anywhere qualifies, select IN_PROGRESS in same item.\n \"\"\"\n create_transcription(asset=self.asset2, user=self.anon) # IN_PROGRESS\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n\n got = find_next_transcribable_campaign_asset(\n self.campaign,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(got, self.asset2)\n\n def test_eligible_base_qs_filters_status_and_published(self):\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n not_started_asset = create_asset(\n item=self.asset1.item, sequence=3, slug=\"tc-more-ns-ok\"\n )\n in_progress_asset = create_asset(\n item=self.asset1.item, sequence=4, slug=\"tc-more-ip-ok\"\n )\n create_transcription(asset=in_progress_asset, user=self.anon) # IN_PROGRESS\n\n other_campaign = create_campaign(slug=\"tc-ebq-c\", title=\"tc-ebq-c\")\n other_project = create_project(\n campaign=other_campaign, slug=\"tc-ebq-p\", title=\"tc-ebq-p\"\n )\n other_item = create_item(project=other_project, item_id=\"tc-ebq-i\")\n other_asset = create_asset(item=other_item, slug=\"tc-ebq-a\")\n\n queryset = tc_eligible_base_qs(self.campaign)\n id_set = set(queryset.values_list(\"id\", flat=True))\n self.assertIn(not_started_asset.id, id_set)\n self.assertIn(in_progress_asset.id, id_set)\n self.assertNotIn(self.asset1.id, id_set)\n self.assertNotIn(other_asset.id, id_set)\n\n def test_cached_transcribable_accessor_returns_rows(self):\n row = NextTranscribableCampaignAsset.objects.create(\n asset=self.asset1,\n campaign=self.campaign,\n item=self.asset1.item,\n item_item_id=self.asset1.item.item_id,\n project=self.asset1.item.project,\n project_slug=self.asset1.item.project.slug,\n sequence=self.asset1.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n queryset = find_cached_transcribable_assets(self.campaign)\n self.assertIn(row.id, queryset.values_list(\"id\", flat=True))\n\n def test_find_in_item_blank_item_id_none(self):\n chosen = tc_find_in_item(self.campaign, item_id=\"\", after_asset_pk=None)\n self.assertIsNone(chosen)\n\n def test_find_ns_in_proj_without_exclude_includes_same_item(self):\n \"\"\"\n exclude_item_id is falsy, so branch where no exclusion is applied.\n Should pick the first NOT_STARTED asset, even if it's in the same item.\n \"\"\"\n project = self.asset1.item.project\n chosen = tc_find_ns_in_proj(self.campaign, project_slug=project.slug)\n self.assertEqual(chosen, self.asset1)\n"
},
{
"path": "concordia/tests/test_utils_next_asset_transcribable_topic.py",
"content": "from unittest.mock import MagicMock, patch\n\nfrom django.test import TestCase\nfrom django.utils.timezone import now\n\nfrom concordia.models import (\n Asset,\n AssetTranscriptionReservation,\n NextTranscribableTopicAsset,\n TranscriptionStatus,\n)\nfrom concordia.utils import get_anonymous_user\nfrom concordia.utils.next_asset import (\n find_new_transcribable_topic_assets,\n find_next_transcribable_topic_asset,\n find_transcribable_topic_asset,\n)\nfrom concordia.utils.next_asset.transcribable.topic import (\n _eligible_transcribable_base_qs as topic_transcribable_eligible_base_qs,\n)\nfrom concordia.utils.next_asset.transcribable.topic import (\n _find_transcribable_in_item_for_topic as topic_find_in_item_for_topic,\n)\nfrom concordia.utils.next_asset.transcribable.topic import (\n _find_transcribable_not_started_in_project_for_topic,\n find_and_order_potential_transcribable_topic_assets,\n find_invalid_next_transcribable_topic_assets,\n)\nfrom concordia.utils.next_asset.transcribable.topic import (\n _next_seq_after as topic_next_seq_after_for_transcribable,\n)\nfrom concordia.utils.next_asset.transcribable.topic import (\n _order_unstarted_first as topic_order_unstarted_first,\n)\nfrom concordia.utils.next_asset.transcribable.topic import (\n _reserved_asset_ids_subq as topic_transcribable_reserved_ids_subq,\n)\n\nfrom .utils import (\n CreateTestUsers,\n create_asset,\n create_campaign,\n create_item,\n create_project,\n create_topic,\n create_transcription,\n)\n\ntopic_find_not_started_in_project_for_topic = (\n _find_transcribable_not_started_in_project_for_topic\n)\nfind_invalid_next_transcribable_topic_assets_fn = (\n find_invalid_next_transcribable_topic_assets\n)\n\n\nclass NextTranscribableTopicAssetTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anon = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(\n slug=\"topic-asset-1\", sequence=1, title=\"Topic Asset 1\"\n )\n self.asset2 = create_asset(\n item=self.asset1.item,\n sequence=2,\n slug=\"topic-asset-2\",\n title=\"Topic Asset 2\",\n )\n self.topic = create_topic(project=self.asset1.item.project)\n\n def test_find_new_transcribable_topic_assets_filters_correctly(self):\n create_transcription(\n asset=self.asset1,\n user=self.anon,\n submitted=now(),\n )\n\n queryset = find_new_transcribable_topic_assets(self.topic)\n self.assertNotIn(self.asset1, queryset)\n assert_in = self.assertIn\n assert_in(self.asset2, queryset)\n\n def test_find_transcribable_topic_asset_from_next_table(self):\n NextTranscribableTopicAsset.objects.create(\n asset=self.asset1,\n topic=self.topic,\n item=self.asset1.item,\n item_item_id=self.asset1.item.item_id,\n project=self.asset1.item.project,\n project_slug=self.asset1.item.project.slug,\n sequence=self.asset1.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n\n asset = find_transcribable_topic_asset(self.topic)\n self.assertEqual(asset, self.asset1)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_find_transcribable_topic_asset_falls_back_and_spawns_task(\n self, mock_get_task\n ):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset = find_transcribable_topic_asset(self.topic)\n self.assertEqual(asset, self.asset1)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_find_next_transcribable_topic_asset_orders_and_falls_back(\n self, mock_get_task\n ):\n \"\"\"\n With short-circuiting: item-level returns the next asset\n and we do not spawn a task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n asset = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(asset, self.asset2)\n # Short-circuit satisfied -> no cache fallback -> no task spawned\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_find_next_transcribable_topic_asset_when_next_asset_exists(\n self, mock_get_task\n ):\n create_transcription(asset=self.asset2, user=self.anon)\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n NextTranscribableTopicAsset.objects.create(\n asset=self.asset2,\n topic=self.topic,\n item=self.asset2.item,\n item_item_id=self.asset2.item.item_id,\n project=self.asset2.item.project,\n project_slug=self.asset2.item.project.slug,\n sequence=self.asset2.sequence,\n transcription_status=TranscriptionStatus.IN_PROGRESS,\n )\n\n asset = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=None,\n )\n self.assertEqual(asset, self.asset2)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n def test_short_circuit_same_item_topic_respects_after_sequence_and_reservations(\n self,\n ):\n third = create_asset(\n item=self.asset1.item,\n sequence=3,\n slug=\"topic-asset-3\",\n title=\"Topic Asset 3\",\n )\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, self.asset2)\n\n AssetTranscriptionReservation.objects.create(\n asset=self.asset2, reservation_token=\"tkn\" # nosec\n )\n chosen2 = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen2, third)\n\n def test_project_short_circuit_topic_prefers_not_started_over_in_progress(self):\n # Exhaust item\n create_transcription(asset=self.asset1, user=self.anon, submitted=now())\n create_transcription(asset=self.asset2, user=self.anon, submitted=now())\n\n other_item = create_item(project=self.asset1.item.project, item_id=\"tproj-item\")\n in_progress = create_asset(\n item=other_item, sequence=1, slug=\"tproj-inprog\", title=\"TProj InProg\"\n )\n create_transcription(asset=in_progress, user=self.anon)\n\n not_started_asset = create_asset(\n item=other_item,\n sequence=2,\n slug=\"tproj-notstarted\",\n title=\"TProj NotStarted\",\n )\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, not_started_asset)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_project_short_circuit_topic_without_item_id_allows_same_item(\n self, mock_get_task\n ):\n \"\"\"\n With item_id not set, the project-level short-circuit\n should return the first NOT_STARTED in the project,\n ordered by (item__item_id, sequence, id).\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=\"\", # falsy -> do not exclude same item\n original_asset_id=None, # no exclusion of original\n )\n # Both assets are NOT_STARTED in the same item; ordering picks asset1.\n self.assertEqual(chosen, self.asset1)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n\nclass TranscribableTopicInternalsTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anonymous = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(sequence=1, slug=\"tt-int-a1\")\n self.asset2 = create_asset(item=self.asset1.item, sequence=2, slug=\"tt-int-a2\")\n self.topic = create_topic(project=self.asset1.item.project)\n\n def test_topic_transcribable_reserved_ids_is_unfiltered(self):\n AssetTranscriptionReservation.objects.create(\n asset=self.asset1, reservation_token=\"tt-res-here\" # nosec\n )\n other_campaign = create_campaign(slug=\"tt-oc\", title=\"tt-oc\")\n other_project = create_project(\n campaign=other_campaign, slug=\"tt-op\", title=\"tt-op\"\n )\n other_item = create_item(project=other_project, item_id=\"tt-oi\")\n other_asset = create_asset(item=other_item, slug=\"tt-oa\")\n AssetTranscriptionReservation.objects.create(\n asset=other_asset, reservation_token=\"tt-res-there\" # nosec\n )\n ids = set(\n topic_transcribable_reserved_ids_subq().values_list(\"asset_id\", flat=True)\n )\n self.assertIn(self.asset1.id, ids)\n self.assertIn(other_asset.id, ids)\n\n def test_topic_transcribable_eligible_base_qs_filters_correctly(self):\n # Submitted, so excluded\n create_transcription(asset=self.asset2, user=self.anonymous, submitted=now())\n # Not started (included)\n asset_not_started = self.asset1\n # In progress (included)\n asset_in_progress = create_asset(\n item=self.asset1.item, sequence=3, slug=\"tt-int-ip\"\n )\n create_transcription(asset=asset_in_progress, user=self.anonymous)\n # Other campaign (excluded)\n other_campaign = create_campaign(slug=\"tt-ebq-c\", title=\"tt-ebq-c\")\n other_project = create_project(\n campaign=other_campaign, slug=\"tt-ebq-p\", title=\"tt-ebq-p\"\n )\n other_item = create_item(project=other_project, item_id=\"tt-ebq-i\")\n other_asset = create_asset(item=other_item, slug=\"tt-ebq-a\")\n queryset = topic_transcribable_eligible_base_qs(self.topic)\n ids = set(queryset.values_list(\"id\", flat=True))\n self.assertIn(asset_not_started.id, ids)\n self.assertIn(asset_in_progress.id, ids)\n self.assertNotIn(self.asset2.id, ids)\n self.assertNotIn(other_asset.id, ids)\n\n def test_topic_next_seq_after_variants_for_transcribable(self):\n self.assertIsNone(topic_next_seq_after_for_transcribable(None))\n self.assertIsNone(topic_next_seq_after_for_transcribable(987654321))\n self.assertEqual(\n topic_next_seq_after_for_transcribable(self.asset2.id),\n self.asset2.sequence,\n )\n\n def test_topic_find_in_item_for_topic_after_none_returns_first_not_started(self):\n chosen = topic_find_in_item_for_topic(\n self.topic, item_id=self.asset1.item.item_id, after_asset_pk=None\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_topic_find_in_item_for_topic_skips_reserved_and_advances(self):\n third = create_asset(item=self.asset1.item, sequence=3, slug=\"tt-int-a3\")\n AssetTranscriptionReservation.objects.create(\n asset=self.asset2, reservation_token=\"tt-int-a2\" # nosec\n )\n chosen = topic_find_in_item_for_topic(\n self.topic,\n item_id=self.asset1.item.item_id,\n after_asset_pk=self.asset1.id,\n )\n self.assertEqual(chosen, third)\n\n def test_topic_find_in_item_for_topic_after_missing_excludes_only_id(self):\n chosen = topic_find_in_item_for_topic(\n self.topic,\n item_id=self.asset1.item.item_id,\n after_asset_pk=987654321,\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_topic_find_in_item_for_topic_blank_item_id_returns_none(self):\n chosen = topic_find_in_item_for_topic(\n self.topic, item_id=\"\", after_asset_pk=None\n )\n self.assertIsNone(chosen)\n\n def test_topic_find_not_started_in_project_excludes_item_and_reserved(self):\n other_item = create_item(project=self.asset1.item.project, item_id=\"tt-int-ex\")\n not_started_1 = create_asset(item=other_item, sequence=1, slug=\"tt-int-ns1\")\n not_started_2 = create_asset(item=other_item, sequence=2, slug=\"tt-int-ns2\")\n AssetTranscriptionReservation.objects.create(\n asset=not_started_1, reservation_token=\"tt-int-res\" # nosec\n )\n chosen = topic_find_not_started_in_project_for_topic(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n exclude_item_id=self.asset1.item.item_id,\n )\n self.assertEqual(chosen, not_started_2)\n\n def test_topic_find_not_started_in_project_blank_slug_none(self):\n self.assertIsNone(\n topic_find_not_started_in_project_for_topic(self.topic, project_slug=\"\")\n )\n\n def test_topic_order_unstarted_first_prefers_not_started(self):\n in_progress = create_asset(\n item=self.asset1.item, sequence=4, slug=\"tt-int-ip-2\"\n )\n create_transcription(asset=in_progress, user=self.anonymous)\n queryset = Asset.objects.filter(id__in=[self.asset1.id, in_progress.id])\n ordered = list(\n topic_order_unstarted_first(queryset).values_list(\"id\", flat=True)\n )\n self.assertEqual(ordered[0], self.asset1.id)\n self.assertEqual(ordered[1], in_progress.id)\n\n def test_topic_find_not_started_in_project_without_exclude_includes_same_item(\n self,\n ):\n \"\"\"\n With exclude_item_id falsy, the helper should consider the same item and\n pick the first NOT_STARTED asset there (covers the else branch of L154->157).\n \"\"\"\n project = self.asset1.item.project\n chosen = topic_find_not_started_in_project_for_topic(\n self.topic, project_slug=project.slug, exclude_item_id=\"\"\n )\n self.assertEqual(chosen, self.asset1)\n\n\nclass NextTranscribableTopicMoreTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.anonymous = get_anonymous_user()\n self.user = self.create_test_user()\n self.asset1 = create_asset(slug=\"tt-more-a1\", sequence=1, title=\"TT More A1\")\n self.asset2 = create_asset(\n item=self.asset1.item, slug=\"tt-more-a2\", sequence=2, title=\"TT More A2\"\n )\n self.topic = create_topic(project=self.asset1.item.project)\n\n def test_new_transcribable_topic_excludes_reserved_and_cached(self):\n reserved_asset = create_asset(\n item=self.asset1.item, sequence=3, slug=\"tt-more-res\"\n )\n cached_asset = create_asset(\n item=self.asset1.item, sequence=4, slug=\"tt-more-cached\"\n )\n NextTranscribableTopicAsset.objects.create(\n asset=cached_asset,\n topic=self.topic,\n item=cached_asset.item,\n item_item_id=cached_asset.item.item_id,\n project=cached_asset.item.project,\n project_slug=cached_asset.item.project.slug,\n sequence=cached_asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n AssetTranscriptionReservation.objects.create(\n asset=reserved_asset, reservation_token=\"tt-more-rv\" # nosec\n )\n queryset = find_new_transcribable_topic_assets(self.topic)\n self.assertNotIn(reserved_asset, queryset)\n self.assertNotIn(cached_asset, queryset)\n\n def test_find_and_order_potential_transcribable_topic_assets_ordering(self):\n base_item = self.asset1.item\n same_item_next = create_asset(\n item=base_item, sequence=10, slug=\"tt-pot-ci-next\"\n )\n same_project_other_item = create_asset(\n item=create_item(project=base_item.project, item_id=\"tt-pot-it-2\"),\n sequence=5,\n slug=\"tt-pot-proj\",\n )\n other_project = create_project(\n campaign=self.asset1.campaign,\n slug=\"tt-pot-proj-oth\",\n title=\"tt-pot-proj-oth\",\n )\n other_item = create_item(project=other_project, item_id=\"tt-pot-it-3\")\n other_project_asset = create_asset(\n item=other_item, sequence=1, slug=\"tt-pot-op\"\n )\n\n for asset in (same_item_next, same_project_other_item, other_project_asset):\n NextTranscribableTopicAsset.objects.create(\n asset=asset,\n topic=self.topic,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n sequence=asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n\n ordered = find_and_order_potential_transcribable_topic_assets(\n self.topic,\n project_slug=base_item.project.slug,\n item_id=base_item.item_id,\n asset_pk=self.asset1.id,\n ).values_list(\"asset_id\", flat=True)\n\n ordered = list(ordered)\n # Prefer same item, then same project, then others\n self.assertEqual(ordered[0], same_item_next.id)\n self.assertEqual(ordered[1], same_project_other_item.id)\n self.assertIn(other_project_asset.id, ordered[2:])\n\n def test_find_invalid_next_transcribable_topic_assets_reserved_and_status(self):\n reserved_asset = create_asset(\n item=self.asset1.item, sequence=30, slug=\"tt-inv-res\"\n )\n create_transcription(asset=reserved_asset, user=self.anonymous)\n AssetTranscriptionReservation.objects.create(\n asset=reserved_asset, reservation_token=\"tt-inv-rv\" # nosec\n )\n NextTranscribableTopicAsset.objects.create(\n asset=reserved_asset,\n topic=self.topic,\n item=reserved_asset.item,\n item_item_id=reserved_asset.item.item_id,\n project=reserved_asset.item.project,\n project_slug=reserved_asset.item.project.slug,\n sequence=reserved_asset.sequence,\n transcription_status=TranscriptionStatus.IN_PROGRESS,\n )\n wrong_status_asset = create_asset(\n item=self.asset1.item, sequence=31, slug=\"tt-inv-wrong\"\n )\n create_transcription(\n asset=wrong_status_asset, user=self.anonymous, submitted=now()\n )\n NextTranscribableTopicAsset.objects.create(\n asset=wrong_status_asset,\n topic=self.topic,\n item=wrong_status_asset.item,\n item_item_id=wrong_status_asset.item.item_id,\n project=wrong_status_asset.item.project,\n project_slug=wrong_status_asset.item.project.slug,\n sequence=wrong_status_asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n bad = list(\n find_invalid_next_transcribable_topic_assets_fn(self.topic.id).values_list(\n \"asset_id\", flat=True\n )\n )\n self.assertIn(reserved_asset.id, bad)\n self.assertIn(wrong_status_asset.id, bad)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_cache_same_item_is_ignored_then_manual_selects_topic(self, mock_get_task):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Same-item cached row should be excluded, forcing manual fallback.\n NextTranscribableTopicAsset.objects.create(\n asset=self.asset2,\n topic=self.topic,\n item=self.asset2.item,\n item_item_id=self.asset2.item.item_id,\n project=self.asset2.item.project,\n project_slug=self.asset2.item.project.slug,\n sequence=self.asset2.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n # Make same-item short-circuit fail by reserving the only candidate.\n AssetTranscriptionReservation.objects.create(\n asset=self.asset2, reservation_token=\"tt-cache-same\" # nosec\n )\n # Provide a valid choice elsewhere to be picked by manual fallback.\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"tt-cache-oth\"\n )\n picked = create_asset(item=other_item, sequence=5, slug=\"tt-cache-pick\")\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=\"\",\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, picked)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_cache_excludes_original_pk_and_chooses_next_topic(self, mock_get_task):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"tt-cache-exc\"\n )\n first = create_asset(item=other_item, sequence=1, slug=\"tt-cache-first\")\n second = create_asset(item=other_item, sequence=2, slug=\"tt-cache-second\")\n for asset in (first, second):\n NextTranscribableTopicAsset.objects.create(\n asset=asset,\n topic=self.topic,\n item=asset.item,\n item_item_id=asset.item.item_id,\n project=asset.item.project,\n project_slug=asset.item.project.slug,\n sequence=asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n\n chosen = find_next_transcribable_topic_asset(\n self.topic, project_slug=\"\", item_id=\"\", original_asset_id=first.id\n )\n self.assertEqual(chosen, second)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n def test_same_item_inprogress_selected_when_no_not_started_topic(self):\n create_transcription(asset=self.asset2, user=self.anonymous)\n create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())\n got = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=None,\n )\n self.assertEqual(got, self.asset2)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_next_transcribable_topic_none_anywhere_returns_none_no_spawn(\n self, mock_get_task\n ):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Use a brand-new topic with no eligible assets anywhere.\n empty_campaign = create_campaign(slug=\"tt-none-c\", title=\"tt-none-c\")\n empty_project = create_project(\n campaign=empty_campaign, slug=\"tt-none-p\", title=\"tt-none-p\"\n )\n empty_topic = create_topic(project=empty_project)\n\n chosen = find_next_transcribable_topic_asset(\n topic=empty_topic, project_slug=\"\", item_id=\"\", original_asset_id=None\n )\n self.assertIsNone(chosen)\n self.assertFalse(mock_get_task.called)\n self.assertFalse(mock_task.delay.called)\n\n def test_item_gate_ignored_when_original_is_other_item_topic(self):\n \"\"\"\n original_asset_id exists but belongs to a different item; item gate is ignored\n and we return the first NOT_STARTED in the requested item\n \"\"\"\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"tt-oth-item\"\n )\n other_asset = create_asset(item=other_item, slug=\"tt-oth-a\")\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=other_asset.id,\n )\n self.assertEqual(chosen, self.asset1)\n\n def test_item_digit_string_missing_treats_as_no_after_topic(self):\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=\"987654321\", # valid digits, missing PK\n )\n self.assertEqual(chosen, self.asset1)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_manual_same_item_ip_when_no_ns_anywhere_topic(self, mock_get_task):\n \"\"\"\n Manual fallback path with item_id present: when there are\n no NOT_STARTED anywhere, choose same-item IN_PROGRESS.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # No NOT_STARTED anywhere in this topic's project; only same-item IN_PROGRESS\n create_transcription(asset=self.asset2, user=self.anonymous) # IN_PROGRESS\n create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())\n\n got = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=\"\", # bypass short-circuit so we hit the manual path\n item_id=self.asset1.item.item_id,\n original_asset_id=None,\n )\n self.assertEqual(got, self.asset2)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n def test_item_invalid_after_str_valueerror_branch_topic(self):\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=\"not-an-int\",\n )\n self.assertEqual(chosen, self.asset1)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_manual_valid_after_excludes_original_and_picks_next_topic(\n self, mock_get_task\n ):\n \"\"\"\n Manual fallback with a valid original_asset_id: use after_seq to exclude the\n original and return the next NOT_STARTED\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"tt-man-item\"\n )\n first = create_asset(item=other_item, sequence=1, slug=\"tt-man-first\")\n second = create_asset(item=other_item, sequence=2, slug=\"tt-man-second\")\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=\"\",\n item_id=\"\",\n original_asset_id=first.id,\n )\n self.assertEqual(chosen, second)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_inprogress_fallback_spawns_and_uses_after_gate_topic(self, mock_get_task):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Make original (seq=1) SUBMITTED so it can't be chosen; keep it as \"original\".\n create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())\n # Only candidate anywhere: same-item IN_PROGRESS (seq=2).\n create_transcription(asset=self.asset2, user=self.anonymous) # IN_PROGRESS\n\n # Ensure there are no other items/assets in the topic to be found\n # by manual path (manual path excludes same item when item_id is provided).\n # No cached rows either.\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n\n self.assertEqual(chosen, self.asset2)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_inprogress_fallback_with_digit_string_original_id(self, mock_get_task):\n \"\"\"\n Same as above, but pass original_asset_id as a DIGIT STRING to\n exercise the int(original_asset_id) path inside the after-seq filter.\n Also ensures exclude(pk=original_asset_id) runs without ValueError.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Original: SUBMITTED, seq=1\n create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())\n # Only candidate: same-item IN_PROGRESS, seq=2\n create_transcription(asset=self.asset2, user=self.anonymous)\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=str(self.asset1.id),\n )\n\n self.assertEqual(chosen, self.asset2)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_inprogress_fallback_spawns_and_returns_asset_topic(self, mock_get_task):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Use a brand-new topic/project so no cached rows can interfere.\n campaign = create_campaign(slug=\"tt-ip-c1\", title=\"tt-ip-c1\")\n project = create_project(\n campaign=campaign,\n slug=\"tt-ip-p1\",\n title=\"tt-ip-p1\",\n )\n topic = create_topic(project=project)\n item = create_item(project=project, item_id=\"tt-ip-i1\")\n\n asset1 = create_asset(item=item, sequence=1, slug=\"tt-ip-a1\")\n asset2 = create_asset(item=item, sequence=2, slug=\"tt-ip-a2\")\n create_transcription(asset=asset1, user=get_anonymous_user(), submitted=now())\n create_transcription(asset=asset2, user=get_anonymous_user())\n\n chosen = find_next_transcribable_topic_asset(\n topic=topic,\n project_slug=project.slug,\n item_id=item.item_id,\n original_asset_id=asset1.id,\n )\n\n self.assertEqual(chosen, asset2)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_inprogress_fallback_with_digit_str_original_id_topic(self, mock_get_task):\n \"\"\"\n Same scenario as above, but pass original_asset_id as a DIGIT STRING to\n run the int(...) path inside the after-seq filter and still spawn the task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n campaign = create_campaign(slug=\"tt-ip-c2\", title=\"tt-ip-c2\")\n project = create_project(\n campaign=campaign,\n slug=\"tt-ip-p2\",\n title=\"tt-ip-p2\",\n )\n topic = create_topic(project=project)\n item = create_item(project=project, item_id=\"tt-ip-i2\")\n\n asset1 = create_asset(item=item, sequence=1, slug=\"tt-ip2-a1\")\n asset2 = create_asset(item=item, sequence=2, slug=\"tt-ip2-a2\")\n create_transcription(asset=asset1, user=get_anonymous_user(), submitted=now())\n create_transcription(asset=asset2, user=get_anonymous_user()) # IN_PROGRESS\n\n chosen = find_next_transcribable_topic_asset(\n topic=topic,\n project_slug=project.slug,\n item_id=item.item_id,\n original_asset_id=str(asset1.id), # digit-string pk\n )\n\n self.assertEqual(chosen, asset2)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n def test_project_short_circuit_topic_excludes_current_item_via_item_filter(self):\n \"\"\"\n project-level short-circuit executes with item_id truthy,\n so the code runs candidate = candidate.exclude(item__item_id=item_id).\n Item-level has no NOT_STARTED, so we land in the project block.\n \"\"\"\n # Exhaust current item (no NOT_STARTED left there)\n create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())\n create_transcription(asset=self.asset2, user=self.anonymous, submitted=now())\n\n # Create a NOT_STARTED candidate in the same project but a different item\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"tt-proj-ex-branch\"\n )\n pick = create_asset(item=other_item, sequence=1, slug=\"tt-proj-ex-pick\")\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, pick)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_inprogress_fallback_spawns_task_with_item_id_topic(self, mock_get_task):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Same item: one IN_PROGRESS, one SUBMITTED -> no NOT_STARTED anywhere.\n create_transcription(asset=self.asset2, user=self.anonymous)\n create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())\n\n # With item_id set, manual fallback excludes same-item candidates, so\n # it returns nothing (spawn_task=True). Then the IN_PROGRESS fallback\n # must return asset2 and trigger the task.\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=None,\n )\n self.assertEqual(chosen, self.asset2)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_manual_same_item_inprogress_triggers_spawn_task_topic(self, mock_get_task):\n \"\"\"\n When there are no NOT_STARTED candidates anywhere (after excluding current\n item/original in the manual fallback), the same-item IN_PROGRESS fallback\n should return an asset AND spawn the cache population task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Same item only:\n # - original (NOT_STARTED) will be excluded by manual fallback,\n # - next is IN_PROGRESS (eligible for final fallback).\n create_transcription(asset=self.asset2, user=self.anonymous)\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=\"\",\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, self.asset2)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_project_short_circuit_excludes_current_item_topic(self, mock_get_task):\n create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())\n create_transcription(asset=self.asset2, user=self.anonymous, submitted=now())\n\n other_item = create_item(\n project=self.asset1.item.project, item_id=\"topic-proj-exclude-item\"\n )\n pick = create_asset(item=other_item, sequence=5, slug=\"topic-proj-exclude-pick\")\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=self.asset1.id,\n )\n self.assertEqual(chosen, pick)\n self.assertFalse(mock_get_task.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_manual_inprogress_fallback_triggers_spawn_task_topic(self, mock_get_task):\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())\n create_transcription(asset=self.asset2, user=self.anonymous)\n\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=None,\n )\n self.assertEqual(chosen, self.asset2)\n self.assertTrue(mock_get_task.called)\n self.assertTrue(mock_task.delay.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_inprogress_fallback_item_id_returns_none_when_no_candidates_topic(\n self, mock_get_task\n ):\n \"\"\"\n With item_id truthy but no same-item IN_PROGRESS (and no NOT_STARTED anywhere),\n the IN_PROGRESS fallback should yield no asset and the function returns None.\n Ensures the path where `asset` is falsy in the IN_PROGRESS block is covered.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Fresh topic/project with a single item and no transcribable assets at all.\n campaign = create_campaign(slug=\"tt-ip-none-c\", title=\"tt-ip-none-c\")\n project = create_project(\n campaign=campaign, slug=\"tt-ip-none-p\", title=\"tt-ip-none-p\"\n )\n topic = create_topic(project=project)\n item = create_item(project=project, item_id=\"tt-ip-none-i\")\n\n a1 = create_asset(item=item, sequence=1, slug=\"tt-ip-none-a1\")\n a2 = create_asset(item=item, sequence=2, slug=\"tt-ip-none-a2\")\n # Make both SUBMITTED so neither NOT_STARTED nor IN_PROGRESS exists.\n create_transcription(asset=a1, user=get_anonymous_user(), submitted=now())\n create_transcription(asset=a2, user=get_anonymous_user(), submitted=now())\n\n got = find_next_transcribable_topic_asset(\n topic=topic,\n project_slug=project.slug,\n item_id=item.item_id, # truthy, so we enter the IN_PROGRESS fallback\n original_asset_id=None,\n )\n self.assertIsNone(got)\n # No task should be spawned since the IN_PROGRESS block found nothing.\n self.assertFalse(mock_get_task.called)\n\n @patch(\"concordia.utils.next_asset.transcribable.topic.get_registered_task\")\n def test_inprogress_fallback_returns_asset_without_spawning(self, mock_get_task):\n \"\"\"\n Force a path where the cache stage is exercised (so spawn_task stays False),\n but yields no usable asset (simulated by making the cached asset retrieval\n return None). The function should then fall through to the IN_PROGRESS\n fallback, return that asset and NOT spawn the population task.\n \"\"\"\n mock_task = mock_get_task.return_value\n mock_task.delay = MagicMock()\n\n # Current item: seq1 SUBMITTED, seq2 IN_PROGRESS -> same-item NOT_STARTED fails.\n create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())\n create_transcription(asset=self.asset2, user=self.anonymous) # IN_PROGRESS\n\n # Prepare a cached candidate in a DIFFERENT project so project short-circuit\n # doesn't grab it (only checks current project_slug).\n other_campaign = create_campaign(slug=\"tt-ip-cache-c\", title=\"tt-ip-cache-c\")\n other_project = create_project(\n campaign=other_campaign, slug=\"tt-ip-cache-p\", title=\"tt-ip-cache-p\"\n )\n other_item = create_item(project=other_project, item_id=\"tt-ip-cache-i\")\n cached_asset = create_asset(item=other_item, sequence=1, slug=\"tt-ip-cache-a\")\n\n NextTranscribableTopicAsset.objects.create(\n asset=cached_asset,\n topic=self.topic,\n item=cached_asset.item,\n item_item_id=cached_asset.item.item_id,\n project=cached_asset.item.project,\n project_slug=cached_asset.item.project.slug,\n sequence=cached_asset.sequence,\n transcription_status=TranscriptionStatus.NOT_STARTED,\n )\n\n # Patch Asset.objects.filter only for id=cached_asset.id so the cache lookup\n # produces an asset_id (spawn_task remains False) but then returns no row,\n # forcing the function into the IN_PROGRESS fallback.\n real_filter = Asset.objects.filter\n\n def filter_side_effect(*args, **kwargs):\n if kwargs == {\"id\": cached_asset.id}:\n qs_mock = MagicMock()\n qs_mock.select_for_update.return_value = qs_mock\n qs_mock.select_related.return_value = qs_mock\n qs_mock.first.return_value = None # simulate skip-locked/missing row\n return qs_mock\n return real_filter(*args, **kwargs)\n\n with patch.object(Asset.objects, \"filter\", side_effect=filter_side_effect):\n chosen = find_next_transcribable_topic_asset(\n self.topic,\n project_slug=self.asset1.item.project.slug,\n item_id=self.asset1.item.item_id,\n original_asset_id=None,\n )\n\n # Should fall back to same-item IN_PROGRESS and NOT spawn the task.\n self.assertEqual(chosen, self.asset2)\n self.assertFalse(mock_get_task.called)\n"
},
{
"path": "concordia/tests/test_validators.py",
"content": "import string\n\nfrom django.core.exceptions import ValidationError\nfrom django.test import TestCase\n\nfrom concordia.passwords.validators import ComplexityValidator\nfrom concordia.validators import DjangoPasswordsValidator\n\n\nclass TestValidators(TestCase):\n def test_DjangoPasswordsValidator(self):\n validator = DjangoPasswordsValidator()\n expected_error = \"Must be more complex (%s)\"\n self.assertIsNone(validator.validate(\"Ab1!\"))\n\n expected_suberror = \"must contain 1 or more unique lowercase characters\"\n with self.assertRaises(ValidationError) as cm:\n validator.validate(\"AB1!\")\n self.assertEqual(cm.exception.messages, [expected_error % expected_suberror])\n self.assertEqual(cm.exception.error_list[0].code, \"complexity\")\n\n expected_suberror = \"must contain 1 or more unique uppercase characters\"\n with self.assertRaises(ValidationError) as cm:\n validator.validate(\"ab1!\")\n self.assertEqual(cm.exception.messages, [expected_error % expected_suberror])\n self.assertEqual(cm.exception.error_list[0].code, \"complexity\")\n\n expected_suberror = \"must contain 1 or more unique digits\"\n with self.assertRaises(ValidationError) as cm:\n validator.validate(\"Ab!\")\n self.assertEqual(cm.exception.messages, [expected_error % expected_suberror])\n self.assertEqual(cm.exception.error_list[0].code, \"complexity\")\n\n expected_suberror = \"must contain 1 or more non unique special characters\"\n with self.assertRaises(ValidationError) as cm:\n validator.validate(\"Ab1\")\n self.assertEqual(cm.exception.messages, [expected_error % expected_suberror])\n self.assertEqual(cm.exception.error_list[0].code, \"complexity\")\n\n self.assertEqual(\n validator.get_help_text(),\n \"Your password fails to meet our complexity requirements.\",\n )\n\n\nclass ComplexityValidatorTests(TestCase):\n def assertValid(self, validator, string):\n try:\n validator(string)\n except ValidationError:\n self.fail(f\"String {string} failed validation unexpectedly\")\n\n def assertInvalid(self, validator, string):\n self.assertRaises(ValidationError, validator, string)\n\n def make_validator(self, **complexities):\n return ComplexityValidator(complexities=complexities)\n\n def test_empty_validator(self):\n validator = ComplexityValidator(complexities=None)\n self.assertValid(validator, \"\")\n\n def test_minimum_uppercase_count(self):\n validator = self.make_validator(UPPER=0)\n self.assertValid(validator, \"no uppercase\")\n self.assertValid(validator, \"Some UpperCase\")\n self.assertValid(validator, \"ALL UPPERCASE\")\n\n validator = self.make_validator(UPPER=1)\n self.assertInvalid(validator, \"no uppercase\")\n self.assertValid(validator, \"Some UpperCase\")\n self.assertValid(validator, \"ALL UPPERCASE\")\n\n validator = self.make_validator(UPPER=100)\n self.assertInvalid(validator, \"no uppercase\")\n self.assertInvalid(validator, \"Some UpperCase\")\n self.assertInvalid(validator, \"ALL UPPERCASE\")\n\n def test_minimum_lowercase_count(self):\n validator = self.make_validator(LOWER=0)\n self.assertValid(validator, \"NO LOWERCASE\")\n self.assertValid(validator, \"sOME lOWERCASE\")\n self.assertValid(validator, \"all lowercase\")\n\n validator = self.make_validator(LOWER=1)\n self.assertInvalid(validator, \"NO LOWERCASE\")\n self.assertValid(validator, \"sOME lOWERCASE\")\n self.assertValid(validator, \"all lowercase\")\n\n validator = self.make_validator(LOWER=100)\n self.assertInvalid(validator, \"NO LOWERCASE\")\n self.assertInvalid(validator, \"sOME lOWERCASE\")\n self.assertInvalid(validator, \"all lowercase\")\n\n def test_minimum_letter_count(self):\n validator = self.make_validator(LETTERS=0)\n self.assertValid(validator, \"1234. ?\")\n self.assertValid(validator, \"soME 123\")\n self.assertValid(validator, \"allletters\")\n\n validator = self.make_validator(LETTERS=1)\n self.assertInvalid(validator, \"1234. ?\")\n self.assertValid(validator, \"soME 123\")\n self.assertValid(validator, \"allletters\")\n\n validator = self.make_validator(LETTERS=100)\n self.assertInvalid(validator, \"1234. ?\")\n self.assertInvalid(validator, \"soME 123\")\n self.assertInvalid(validator, \"allletters\")\n\n def test_minimum_digit_count(self):\n validator = self.make_validator(DIGITS=0)\n self.assertValid(validator, \"\")\n self.assertValid(validator, \"0\")\n self.assertValid(validator, \"1\")\n self.assertValid(validator, \"11\")\n self.assertValid(validator, \"one 1\")\n\n validator = self.make_validator(DIGITS=1)\n self.assertInvalid(validator, \"\")\n self.assertValid(validator, \"0\")\n self.assertValid(validator, \"1\")\n self.assertValid(validator, \"11\")\n self.assertValid(validator, \"one 1\")\n\n def test_minimum_punctuation_count(self):\n none = \"no punctuation\"\n one = \"ffs!\"\n mixed = r\"w@oo%lo(om!ol~oo&\"\n allpunc = string.punctuation\n\n validator = self.make_validator(SPECIAL=0)\n self.assertValid(validator, none)\n self.assertValid(validator, one)\n self.assertValid(validator, mixed)\n self.assertValid(validator, allpunc)\n\n validator = self.make_validator(SPECIAL=1)\n self.assertInvalid(validator, none)\n self.assertValid(validator, one)\n self.assertValid(validator, mixed)\n self.assertValid(validator, allpunc)\n\n validator = self.make_validator(SPECIAL=100)\n self.assertInvalid(validator, none)\n self.assertInvalid(validator, one)\n self.assertInvalid(validator, mixed)\n self.assertInvalid(validator, allpunc)\n\n def test_minimum_nonascii_count(self):\n none = \"regularchars and numbers 100\"\n one = \"\\x00\" # null\n many = \"\\x00\\x01\\x02\\x03\\x04\\x05\\t\\n\\r\"\n\n validator = self.make_validator(SPECIAL=0)\n self.assertValid(validator, none)\n self.assertValid(validator, one)\n self.assertValid(validator, many)\n\n validator = self.make_validator(SPECIAL=1)\n self.assertInvalid(validator, none)\n self.assertValid(validator, one)\n self.assertValid(validator, many)\n\n validator = self.make_validator(SPECIAL=100)\n self.assertInvalid(validator, none)\n self.assertInvalid(validator, one)\n self.assertInvalid(validator, many)\n\n def test_minimum_words_count(self):\n none = \"\"\n one = \"oneword\"\n some = \"one or two words\"\n many = \"a b c d e f g h i 1 2 3 4 5 6 7 8 9 { $ # ! )}\"\n\n validator = self.make_validator(WORDS=0)\n self.assertValid(validator, none)\n self.assertValid(validator, one)\n self.assertValid(validator, some)\n self.assertValid(validator, many)\n\n validator = self.make_validator(WORDS=1)\n self.assertInvalid(validator, none)\n self.assertValid(validator, one)\n self.assertValid(validator, some)\n self.assertValid(validator, many)\n\n validator = self.make_validator(WORDS=10)\n self.assertInvalid(validator, none)\n self.assertInvalid(validator, one)\n self.assertInvalid(validator, some)\n self.assertValid(validator, many)\n\n validator = self.make_validator(WORDS=100)\n self.assertInvalid(validator, none)\n self.assertInvalid(validator, one)\n self.assertInvalid(validator, some)\n self.assertInvalid(validator, many)\n"
},
{
"path": "concordia/tests/test_view_decorators.py",
"content": "from unittest.mock import MagicMock, patch\n\nfrom django.core.exceptions import ObjectDoesNotExist, ValidationError\nfrom django.http import HttpRequest\nfrom django.test import TestCase\n\nfrom concordia.views.decorators import next_asset_rate\n\n\nclass TestNextAssetRate(TestCase):\n def setUp(self):\n self.request = HttpRequest()\n\n def test_authenticated_user_returns_none(self):\n self.request.user = MagicMock(is_authenticated=True)\n result = next_asset_rate(\"any.group\", self.request)\n self.assertIsNone(result)\n\n @patch(\"concordia.views.decorators.configuration_value\")\n @patch(\"concordia.views.decorators.validate_rate\")\n def test_anonymous_user_valid_rate(self, mock_validate_rate, mock_config_value):\n self.request.user = MagicMock(is_authenticated=False)\n mock_config_value.return_value = \"10/m\"\n mock_validate_rate.return_value = \"10/m\"\n\n result = next_asset_rate(\"next_asset\", self.request)\n self.assertEqual(result, \"10/m\")\n\n @patch(\"concordia.views.decorators.configuration_value\")\n @patch(\"concordia.views.decorators.validate_rate\")\n def test_anonymous_user_invalid_rate_falls_back(\n self, mock_validate_rate, mock_config_value\n ):\n self.request.user = MagicMock(is_authenticated=False)\n mock_config_value.return_value = \"invalid\"\n mock_validate_rate.side_effect = ValidationError(\"bad\")\n\n result = next_asset_rate(\"next_asset\", self.request)\n self.assertEqual(result, \"4/m\")\n\n @patch(\"concordia.views.decorators.configuration_value\")\n def test_anonymous_user_missing_value_falls_back(self, mock_config_value):\n self.request.user = MagicMock(is_authenticated=False)\n mock_config_value.side_effect = ObjectDoesNotExist()\n\n result = next_asset_rate(\"next_asset\", self.request)\n self.assertEqual(result, \"4/m\")\n"
},
{
"path": "concordia/tests/test_views.py",
"content": "import json\nfrom datetime import date, timedelta\nfrom unittest.mock import patch\n\nfrom django import forms\nfrom django.contrib.auth.models import AnonymousUser\nfrom django.core.cache import caches\nfrom django.db.models.signals import post_save\nfrom django.http import HttpResponse, JsonResponse\nfrom django.test import (\n Client,\n RequestFactory,\n TestCase,\n override_settings,\n)\nfrom django.urls import reverse\nfrom django.utils.decorators import method_decorator\nfrom django.utils.timezone import now\n\nfrom concordia.models import (\n Asset,\n Campaign,\n Transcription,\n)\nfrom concordia.signals.handlers import on_transcription_save\nfrom concordia.tasks.reports.sitereport import campaign_report\nfrom concordia.utils import get_anonymous_user\nfrom concordia.views.accounts import AccountProfileView, registration_rate\nfrom concordia.views.campaigns import CompletedCampaignListView\nfrom concordia.views.decorators import reserve_rate, user_cache_control\nfrom concordia.views.items import FilteredItemDetailView\nfrom concordia.views.projects import FilteredProjectDetailView\nfrom concordia.views.rate_limit import ratelimit_view\nfrom concordia.views.visualizations import VisualizationDataView\n\nfrom .utils import (\n CreateTestUsers,\n JSONAssertMixin,\n create_asset,\n create_campaign,\n create_card_family,\n create_guide,\n create_item,\n create_project,\n create_research_center,\n create_tag_collection,\n create_topic,\n create_transcription,\n)\n\n\ndef setup_view(view, request, user=None, *args, **kwargs):\n \"\"\"\n https://stackoverflow.com/a/33647251/10320488\n \"\"\"\n if user:\n request.user = user\n view.request = request\n view.args = args\n view.kwargs = kwargs\n return view\n\n\nclass AccountProfileViewTests(CreateTestUsers, TestCase):\n \"\"\"\n This class contains the unit tests for the AccountProfileView.\n \"\"\"\n\n def test_get_queryset(self):\n \"\"\"\n Test the get_queryset method\n \"\"\"\n self.login_user()\n v = setup_view(\n AccountProfileView(),\n RequestFactory().get(\"account/password_reset/\"),\n user=self.user,\n )\n qs = v.get_queryset()\n self.assertEqual(qs.count(), 0)\n\n\nclass CompletedCampaignListViewTests(TestCase):\n \"\"\"\n This class contains the unit tests for the CompletedCampaignListView\n \"\"\"\n\n def setUp(self):\n today = date.today()\n yesterday = today - timedelta(days=1)\n\n self.research_center = create_research_center()\n self.campaign2 = create_campaign(\n published=True,\n status=Campaign.Status.COMPLETED,\n slug=\"test-campaign-2\",\n completed_date=yesterday,\n )\n self.campaign2.research_centers.add(self.research_center)\n self.campaign3 = create_campaign(\n published=True,\n status=Campaign.Status.RETIRED,\n slug=\"test-campaign-3\",\n completed_date=yesterday,\n )\n\n def test_get_all_campaigns(self):\n active = create_campaign(\n published=True,\n slug=\"test-campaign-4\",\n completed_date=self.campaign2.completed_date,\n )\n view = CompletedCampaignListView()\n view.request = RequestFactory().get(\"/campaigns/completed/\")\n completed_and_retired = view._get_all_campaigns()\n self.assertNotIn(active, completed_and_retired)\n self.assertIn(self.campaign2, completed_and_retired)\n self.assertIn(self.campaign3, completed_and_retired)\n\n view.request = RequestFactory().get(\"/campaigns/completed/?type=completed\")\n completed_campaigns = view._get_all_campaigns()\n self.assertNotIn(active, completed_campaigns)\n self.assertIn(self.campaign2, completed_campaigns)\n self.assertNotIn(self.campaign3, completed_campaigns)\n\n view.request = RequestFactory().get(\"/campaigns/completed/?type=retired\")\n retired_campaigns = view._get_all_campaigns()\n self.assertNotIn(active, retired_campaigns)\n self.assertNotIn(self.campaign2, retired_campaigns)\n self.assertIn(self.campaign3, retired_campaigns)\n\n def test_queryset(self):\n today = date.today()\n create_campaign(\n published=True, status=Campaign.Status.COMPLETED, completed_date=today\n )\n\n view = CompletedCampaignListView()\n\n # Test default\n view.request = RequestFactory().get(\"/campaigns/completed/\")\n queryset = view.get_queryset()\n self.assertGreater(\n queryset.first().completed_date, queryset.last().completed_date\n )\n\n # Test retired\n view.request = RequestFactory().get(\"/campaigns/completed/?type=retired\")\n queryset = view.get_queryset()\n self.assertEqual(queryset.count(), 1)\n\n def test_context_data(self):\n request = RequestFactory().get(\"/campaigns/completed/\")\n response = CompletedCampaignListView.as_view()(request)\n self.assertIsInstance(response.context_data, dict)\n self.assertEqual(response.context_data[\"result_count\"], 2)\n\n request = RequestFactory().get(\"/campaigns/completed/?type=completed\")\n response = CompletedCampaignListView.as_view()(request)\n self.assertIsInstance(response.context_data, dict)\n self.assertEqual(response.context_data[\"result_count\"], 1)\n\n request = RequestFactory().get(\"/campaigns/completed/?type=completed\")\n response = CompletedCampaignListView.as_view()(request)\n self.assertIsInstance(response.context_data, dict)\n self.assertEqual(response.context_data[\"result_count\"], 1)\n\n request = RequestFactory().get(\n f\"/campaigns/completed/?research_center={self.research_center.id}\"\n )\n response = CompletedCampaignListView.as_view()(request)\n self.assertIsInstance(response.context_data, dict)\n self.assertEqual(response.context_data[\"result_count\"], 1)\n\n def test_research_centers(self):\n today = date.today()\n\n create_campaign(\n published=True, status=Campaign.Status.COMPLETED, completed_date=today\n )\n\n url = f\"/campaigns/completed/?research_center={self.research_center.id}\"\n\n # Test queryset directly\n view = CompletedCampaignListView()\n view.request = RequestFactory().get(url)\n queryset = view.get_queryset()\n\n self.assertEqual(queryset.count(), 1)\n\n # Test get_context_data through a get\n response = self.client.get(url)\n\n self.assertIn(\"research_centers\", response.context)\n self.assertEqual(response.context[\"research_centers\"][0], self.research_center)\n\n\n@override_settings(\n RATELIMIT_ENABLE=False, SESSION_ENGINE=\"django.contrib.sessions.backends.cache\"\n)\nclass ConcordiaViewTests(CreateTestUsers, JSONAssertMixin, TestCase):\n \"\"\"\n This class contains the unit tests for the view in the concordia app.\n \"\"\"\n\n def setUp(self):\n for cache in caches.all():\n cache.clear()\n\n def tearDown(self):\n for cache in caches.all():\n cache.clear()\n\n def test_ratelimit_view(self):\n c = Client()\n response = c.get(\"/error/429/\")\n self.assertIsInstance(response, HttpResponse)\n self.assertEqual(response.status_code, 429)\n\n headers = {\"HTTP_X_REQUESTED_WITH\": \"XMLHttpRequest\"}\n response = c.get(\"/error/429/\", **headers)\n self.assertIsInstance(response, JsonResponse)\n self.assertEqual(response.status_code, 429)\n\n def test_campaign_topic_list_view(self):\n \"\"\"\n Test the GET method for route /campaigns-topics\n \"\"\"\n campaign = create_campaign(title=\"Hello Everyone\")\n topic_project = create_project(campaign=campaign)\n campaign_item = create_item(project=topic_project)\n create_asset(item=campaign_item)\n unlisted_campaign = create_campaign(\n title=\"Hello to only certain people\", unlisted=True\n )\n unlisted_topic_project = create_project(campaign=unlisted_campaign)\n unlisted_campaign_item = create_item(project=unlisted_topic_project)\n create_asset(item=unlisted_campaign_item)\n topic = create_topic(title=\"A Listed Topic\", project=topic_project)\n unlisted_topic = create_topic(\n title=\"An Unlisted Topic\", unlisted=True, project=unlisted_topic_project\n )\n\n response = self.client.get(reverse(\"campaign-topic-list\"))\n\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/campaign_topic_list.html\"\n )\n self.assertContains(response, topic.title)\n self.assertNotContains(response, unlisted_topic.title)\n self.assertContains(response, campaign.title)\n self.assertNotContains(response, unlisted_campaign.title)\n\n def test_campaign_list_view(self):\n \"\"\"\n Test the GET method for route /campaigns\n \"\"\"\n campaign = create_campaign(title=\"Hello Everyone 2\")\n unlisted_campaign = create_campaign(\n title=\"Hello to only certain people 2\", unlisted=True\n )\n\n response = self.client.get(reverse(\"transcriptions:campaign-list\"))\n\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/campaign_list.html\"\n )\n self.assertContains(response, campaign.title)\n self.assertNotContains(response, unlisted_campaign.title)\n\n def test_campaign_detail_view(self):\n \"\"\"\n Test GET on route /campaigns/ (campaign)\n \"\"\"\n campaign = create_campaign(title=\"GET Campaign\", slug=\"get-campaign\")\n response = self.client.get(\n reverse(\"transcriptions:campaign-detail\", args=(campaign.slug,))\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/campaign_detail.html\"\n )\n self.assertContains(response, campaign.title)\n # Filter by reviewable parameter check\n response = self.client.get(\n reverse(\"transcriptions:campaign-detail\", args=(campaign.slug,)),\n {\"filter_by_reviewable\": True},\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/campaign_detail.html\"\n )\n self.assertContains(response, campaign.title)\n # Bad status parameter check\n response = self.client.get(\n reverse(\"transcriptions:campaign-detail\", args=(campaign.slug,)),\n {\"transcription_status\": \"bad_parameter\"},\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/campaign_detail.html\"\n )\n self.assertContains(response, campaign.title)\n\n # Unlisted\n campaign = create_campaign(\n title=\"GET Unlisted Campaign\", unlisted=True, slug=\"get-unlisted-campaign\"\n )\n response = self.client.get(\n reverse(\"transcriptions:campaign-detail\", args=(campaign.slug,))\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/campaign_detail.html\"\n )\n self.assertContains(response, campaign.title)\n\n # Completed\n campaign = create_campaign(\n title=\"GET Completed Campaign\",\n slug=\"get-completed-campaign\",\n status=Campaign.Status.COMPLETED,\n )\n response = self.client.get(\n reverse(\"transcriptions:campaign-detail\", args=(campaign.slug,))\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/campaign_detail_completed.html\"\n )\n self.assertContains(response, campaign.title)\n\n # Retired\n campaign = create_campaign(\n title=\"GET Retired Campaign\",\n slug=\"get-retired-campaign\",\n status=Campaign.Status.RETIRED,\n )\n # We need a site report for a retired campaign because\n # that's where the view pulls data from\n campaign_report(campaign=campaign)\n response = self.client.get(\n reverse(\"transcriptions:campaign-detail\", args=(campaign.slug,))\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/campaign_detail_retired.html\"\n )\n self.assertContains(response, campaign.title)\n\n def test_campaign_unicode_slug(self):\n \"\"\"Confirm that Unicode characters are usable in Campaign URLs\"\"\"\n\n campaign = create_campaign(title=\"δ½ ε₯½ World\")\n\n self.assertEqual(campaign.slug, \"δ½ ε₯½-world\")\n\n response = self.client.get(campaign.get_absolute_url())\n\n self.assertEqual(response.status_code, 200)\n\n def test_concordiaCampaignView_get_page2(self):\n \"\"\"\n Test GET on route /campaigns// (campaign) on page 2\n \"\"\"\n c = create_campaign()\n\n response = self.client.get(\n reverse(\"transcriptions:campaign-detail\", args=(c.slug,)), {\"page\": 2}\n )\n\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/campaign_detail.html\"\n )\n\n def test_empty_item_detail_view(self):\n \"\"\"\n Test item detail display with no assets\n \"\"\"\n\n item = create_item()\n\n response = self.client.get(\n reverse(\n \"transcriptions:item-detail\",\n args=(item.project.campaign.slug, item.project.slug, item.item_id),\n )\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/item_detail.html\"\n )\n self.assertContains(response, item.title)\n self.assertEqual(0, response.context[\"not_started_percent\"])\n self.assertEqual(0, response.context[\"in_progress_percent\"])\n self.assertEqual(0, response.context[\"submitted_percent\"])\n self.assertEqual(0, response.context[\"completed_percent\"])\n\n def test_item_detail_view(self):\n \"\"\"\n Test item detail display with assets\n \"\"\"\n\n self.login_user() # Implicitly create the test account\n anon = get_anonymous_user()\n\n item = create_item()\n # We'll create 10 assets and transcriptions for some of them so we can\n # confirm that the math is working correctly:\n for i in range(1, 11):\n asset = create_asset(item=item, sequence=i, slug=f\"test-{i}\")\n if i > 9:\n t = asset.transcription_set.create(asset=asset, user=anon)\n t.submitted = now()\n t.accepted = now()\n t.reviewed_by = self.user\n elif i > 7:\n t = asset.transcription_set.create(asset=asset, user=anon)\n t.submitted = now()\n elif i > 4:\n t = asset.transcription_set.create(asset=asset, user=anon)\n else:\n continue\n\n t.full_clean()\n t.save()\n\n response = self.client.get(\n reverse(\n \"transcriptions:item-detail\",\n args=(item.project.campaign.slug, item.project.slug, item.item_id),\n )\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/item_detail.html\"\n )\n self.assertContains(response, item.title)\n # We have 10 total, 6 of which have transcription records and of those\n # 6, 3 have been submitted and one of those was accepted:\n self.assertEqual(40, response.context[\"not_started_percent\"])\n self.assertEqual(30, response.context[\"in_progress_percent\"])\n self.assertEqual(20, response.context[\"submitted_percent\"])\n self.assertEqual(10, response.context[\"completed_percent\"])\n # Filter by reviewable parameter check\n response = self.client.get(\n reverse(\n \"transcriptions:item-detail\",\n args=(item.project.campaign.slug, item.project.slug, item.item_id),\n ),\n {\"filter_by_reviewable\": True},\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/item_detail.html\"\n )\n # Bad status parameter check\n response = self.client.get(\n reverse(\n \"transcriptions:item-detail\",\n args=(item.project.campaign.slug, item.project.slug, item.item_id),\n ),\n {\"transcription_status\": \"bad_parameter\"},\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/item_detail.html\"\n )\n\n # Non-existent item in an existing campaign\n response = self.client.get(\n reverse(\n \"transcriptions:item-detail\",\n args=(item.project.campaign.slug, item.project.slug, \"bad-id\"),\n )\n )\n self.assertRedirects(\n response,\n reverse(\n \"transcriptions:campaign-detail\", args=(item.project.campaign.slug,)\n ),\n )\n\n def test_asset_unicode_slug(self):\n \"\"\"Confirm that Unicode characters are usable in Asset URLs\"\"\"\n\n asset = create_asset(title=\"δ½ ε₯½ World\")\n\n self.assertEqual(asset.slug, \"δ½ ε₯½-world\")\n\n response = self.client.get(asset.get_absolute_url())\n\n self.assertEqual(response.status_code, 200)\n\n def test_asset_detail_view(self):\n \"\"\"\n This unit test test the GET route /campaigns//asset//\n with already in use.\n \"\"\"\n self.login_user()\n\n asset = create_asset(sequence=100)\n\n self.transcription = asset.transcription_set.create(\n user_id=self.user.id, text=\"Test transcription 1\"\n )\n self.transcription.save()\n\n asset.item.project.campaign.card_family = create_card_family()\n asset.item.project.campaign.save()\n title = \"Transcription: Basic Rules\"\n create_guide(title=title)\n\n tag_collection = create_tag_collection(asset=asset)\n\n response = self.client.get(\n reverse(\n \"transcriptions:asset-detail\",\n kwargs={\n \"campaign_slug\": asset.item.project.campaign.slug,\n \"project_slug\": asset.item.project.slug,\n \"item_id\": asset.item.item_id,\n \"slug\": asset.slug,\n },\n )\n )\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"cards\", response.context)\n self.assertIn(\"guides\", response.context)\n self.assertEqual(title, response.context[\"guides\"][0][\"title\"])\n self.assertIn(\"tags\", response.context)\n self.assertEqual([tag_collection.tags.all()[0].value], response.context[\"tags\"])\n\n # Next and previous asset checks\n previous_asset = create_asset(\n item=asset.item, slug=\"previous-asset\", sequence=1\n )\n next_asset = create_asset(item=asset.item, slug=\"next-asset\", sequence=1000)\n response = self.client.get(\n reverse(\n \"transcriptions:asset-detail\",\n kwargs={\n \"campaign_slug\": asset.item.project.campaign.slug,\n \"project_slug\": asset.item.project.slug,\n \"item_id\": asset.item.item_id,\n \"slug\": asset.slug,\n },\n )\n )\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"previous_asset_url\", response.context)\n self.assertEqual(\n previous_asset.get_absolute_url(), response.context[\"previous_asset_url\"]\n )\n self.assertIn(\"next_asset_url\", response.context)\n self.assertEqual(\n next_asset.get_absolute_url(), response.context[\"next_asset_url\"]\n )\n\n # Download URL iiif check\n asset.download_url = \"http://tile.loc.gov/image-services/iiif/service:music:mussuffrage:mussuffrage-100183:mussuffrage-100183.0001/full/pct:100/0/default.jpg\"\n asset.save()\n response = self.client.get(\n reverse(\n \"transcriptions:asset-detail\",\n kwargs={\n \"campaign_slug\": asset.item.project.campaign.slug,\n \"project_slug\": asset.item.project.slug,\n \"item_id\": asset.item.item_id,\n \"slug\": asset.slug,\n },\n )\n )\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"thumbnail_url\", response.context)\n self.assertEqual(\n \"https://tile.loc.gov/image-services/iiif/service:music:mussuffrage:mussuffrage-100183:mussuffrage-100183.0001/full/!512,512/0/default.jpg\",\n response.context[\"thumbnail_url\"],\n )\n\n # Non-existent asset in an existing campaign\n response = self.client.get(\n reverse(\n \"transcriptions:asset-detail\",\n kwargs={\n \"campaign_slug\": asset.item.project.campaign.slug,\n \"project_slug\": asset.item.project.slug,\n \"item_id\": asset.item.item_id,\n \"slug\": \"bad-slug\",\n },\n )\n )\n self.assertRedirects(\n response,\n reverse(\n \"transcriptions:campaign-detail\",\n args=(asset.item.project.campaign.slug,),\n ),\n )\n\n @patch.object(Asset, \"get_ocr_transcript\")\n def test_generate_ocr_transcription(self, mock):\n asset1 = create_asset(storage_image=\"tests/test-european.jpg\")\n url = reverse(\"generate-ocr-transcription\", kwargs={\"asset_pk\": asset1.pk})\n\n # Anonymous user test; should redirect\n response = self.client.post(url)\n self.assertEqual(302, response.status_code)\n self.assertFalse(mock.called)\n mock.reset_mock()\n\n self.login_user()\n response = self.client.post(url)\n self.assertEqual(201, response.status_code)\n self.assertTrue(mock.called)\n mock.reset_mock()\n\n asset2 = create_asset(\n item=asset1.item,\n slug=\"test-asset-2\",\n storage_image=\"tests/test-european.jpg\",\n )\n url = reverse(\"generate-ocr-transcription\", kwargs={\"asset_pk\": asset2.pk})\n response = self.client.post(url, data={\"language\": \"spa\"})\n self.assertEqual(201, response.status_code)\n mock.assert_called_with(\"spa\")\n mock.reset_mock()\n\n with patch(\n \"concordia.views.ajax.get_transcription_superseded\"\n ) as superseded_mock:\n # Test case if the trancription being replaced has already been superseded\n superseded_mock.return_value = HttpResponse(status=409)\n url = reverse(\"generate-ocr-transcription\", kwargs={\"asset_pk\": asset2.pk})\n response = self.client.post(url)\n self.assertEqual(409, response.status_code)\n self.assertTrue(superseded_mock.called)\n self.assertFalse(mock.called)\n\n # Test case if the transcription being replaced hasn't been superseded\n superseded_mock.reset_mock()\n superseded_mock.return_value = create_transcription(\n asset=asset2, user=get_anonymous_user(), submitted=now()\n )\n url = reverse(\"generate-ocr-transcription\", kwargs={\"asset_pk\": asset2.pk})\n response = self.client.post(url)\n self.assertEqual(201, response.status_code)\n self.assertTrue(superseded_mock.called)\n self.assertTrue(mock.called)\n\n def test_project_detail_view(self):\n \"\"\"\n Test GET on route /campaigns/ (campaign)\n \"\"\"\n project = create_project()\n\n response = self.client.get(\n reverse(\n \"transcriptions:project-detail\",\n args=(project.campaign.slug, project.slug),\n )\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/project_detail.html\"\n )\n # Filter by reviewable parameter check\n response = self.client.get(\n reverse(\n \"transcriptions:project-detail\",\n args=(project.campaign.slug, project.slug),\n ),\n {\"filter_by_reviewable\": True},\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/project_detail.html\"\n )\n # Bad status parameter check\n response = self.client.get(\n reverse(\n \"transcriptions:project-detail\",\n args=(project.campaign.slug, project.slug),\n ),\n {\"transcription_status\": \"bad_parameter\"},\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/project_detail.html\"\n )\n\n # Non-existent project in an existing campaign\n response = self.client.get(\n reverse(\n \"transcriptions:project-detail\",\n args=(project.campaign.slug, \"bad-slug\"),\n )\n )\n self.assertRedirects(\n response,\n reverse(\"transcriptions:campaign-detail\", args=(project.campaign.slug,)),\n )\n\n def test_project_unicode_slug(self):\n \"\"\"Confirm that Unicode characters are usable in Project URLs\"\"\"\n\n project = create_project(title=\"δ½ ε₯½ World\")\n\n self.assertEqual(project.slug, \"δ½ ε₯½-world\")\n\n response = self.client.get(project.get_absolute_url())\n\n self.assertEqual(response.status_code, 200)\n\n def test_campaign_report(self):\n \"\"\"\n Test campaign reporting\n \"\"\"\n\n item = create_item()\n # We'll create 10 assets and transcriptions for some of them so we can\n # confirm that the math is working correctly:\n for i in range(1, 11):\n create_asset(item=item, sequence=i, slug=f\"test-{i}\")\n\n response = self.client.get(\n reverse(\n \"transcriptions:campaign-report\",\n kwargs={\"campaign_slug\": item.project.campaign.slug},\n )\n )\n\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(response, \"transcriptions/campaign_report.html\")\n\n ctx = response.context\n\n self.assertEqual(ctx[\"title\"], item.project.campaign.title)\n self.assertEqual(ctx[\"total_asset_count\"], 10)\n\n response = self.client.get(\n reverse(\n \"transcriptions:campaign-report\",\n kwargs={\"campaign_slug\": item.project.campaign.slug},\n ),\n {\"page\": \"not-an-int\"},\n )\n\n ctx = response.context\n\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(response, \"transcriptions/campaign_report.html\")\n self.assertEqual(ctx[\"projects\"].number, 1)\n\n response = self.client.get(\n reverse(\n \"transcriptions:campaign-report\",\n kwargs={\"campaign_slug\": item.project.campaign.slug},\n ),\n {\"page\": 10000},\n )\n\n ctx = response.context\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(response, \"transcriptions/campaign_report.html\")\n self.assertEqual(ctx[\"projects\"].number, 1)\n\n\nclass UserCacheControlTest(CreateTestUsers, TestCase):\n \"\"\"\n Tests for the user_cache_control decorator\n \"\"\"\n\n def setUp(self):\n self.factory = RequestFactory()\n self.user = self.create_user(\"testuser\")\n\n def test_vary_on_cookie(self):\n @method_decorator(user_cache_control, name=\"dispatch\")\n def a_view(request):\n return HttpResponse()\n\n request = self.factory.get(\"/rand\")\n request.user = self.user\n resp = a_view(None, request)\n self.assertEqual(resp.status_code, 200)\n\n\nclass FilteredCampaignDetailViewTests(CreateTestUsers, TestCase):\n def test_get_context_data(self):\n campaign = create_campaign()\n kwargs = {\"slug\": campaign.slug}\n url = reverse(\"transcriptions:filtered-campaign-detail\", kwargs=kwargs)\n\n self.login_user(is_staff=False)\n response = self.client.get(url, kwargs)\n self.assertFalse(response.context.get(\"filter_by_reviewable\", False))\n self.logout_user()\n\n self.user = self.create_staff_user()\n self.login_user()\n response = self.client.get(url, kwargs)\n self.assertTrue(response.context.get(\"filter_by_reviewable\"))\n\n\nclass FilteredProjectDetailViewTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.project = create_project()\n self.kwargs = {\n \"campaign_slug\": self.project.campaign.slug,\n \"slug\": self.project.slug,\n }\n self.url = reverse(\"transcriptions:filtered-project-detail\", kwargs=self.kwargs)\n self.login_user()\n\n def test_get_queryset(self):\n item1 = create_item(project=self.project, item_id=\"testitem.012345679\")\n asset1 = create_asset(item=item1)\n create_transcription(asset=asset1, user=get_anonymous_user(), submitted=now())\n\n item2 = create_item(\n project=create_project(slug=\"project-two\", campaign=self.project.campaign)\n )\n asset2 = create_asset(item=item2)\n create_transcription(asset=asset2, user=self.user, submitted=now())\n\n view = FilteredProjectDetailView()\n view.kwargs = self.kwargs\n view.request = RequestFactory().get(self.url, self.kwargs)\n view.request.user = self.user\n qs = view.get_queryset()\n self.assertIn(item1, qs)\n self.assertNotIn(item2, qs)\n\n def test_get_context_data(self):\n response = self.client.get(self.url, self.kwargs)\n self.assertTrue(response.context.get(\"filter_by_reviewable\"))\n\n def tearDown(self):\n post_save.connect(on_transcription_save, sender=Transcription)\n\n\nclass FilteredItemDetailViewTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.item = create_item()\n self.kwargs = {\n \"campaign_slug\": self.item.project.campaign.slug,\n \"project_slug\": self.item.project.slug,\n \"item_id\": self.item.item_id,\n }\n self.url = reverse(\"transcriptions:filtered-item-detail\", kwargs=self.kwargs)\n self.login_user()\n\n def test_get_queryset(self):\n asset1 = create_asset(item=self.item)\n create_transcription(asset=asset1, user=get_anonymous_user(), submitted=now())\n\n asset2 = create_asset(item=self.item, slug=\"asset-two\")\n create_transcription(asset=asset2, user=self.user, submitted=now())\n\n view = FilteredItemDetailView()\n view.kwargs = self.kwargs\n view.request = RequestFactory().get(self.url, self.kwargs)\n view.request.user = self.user\n qs = view.get_queryset()\n self.assertIn(asset1, qs)\n self.assertNotIn(asset2, qs)\n\n def test_get_context_data(self):\n response = self.client.get(self.url, self.kwargs)\n self.assertTrue(response.context.get(\"filter_by_reviewable\"))\n\n def tearDown(self):\n post_save.connect(on_transcription_save, sender=Transcription)\n\n\nclass RateLimitTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.request_factory = RequestFactory()\n self.user = self.create_user(\"test-user\")\n\n def test_registration_rate(self):\n request = self.request_factory.get(\"/\")\n self.assertEqual(registration_rate(None, request), \"10/h\")\n with patch(\"concordia.views.accounts.UserRegistrationForm\", autospec=True):\n # This causes the form to test as valid even though there's no data\n self.assertIsNone(registration_rate(None, request))\n\n def test_ratelimit_view(self):\n request = self.request_factory.post(\"/\")\n exception = Exception()\n response = ratelimit_view(request, exception)\n self.assertEqual(response.status_code, 429)\n self.assertNotEqual(response[\"Retry-After\"], 0)\n\n def test_reserve_rate(self):\n request = self.request_factory.post(\"/\")\n\n request.user = AnonymousUser()\n self.assertEqual(\"100/m\", reserve_rate(\"test.group\", request))\n\n request.user = self.user\n self.assertEqual(None, reserve_rate(\"test.group\", request))\n\n\nclass LoginTests(TestCase, CreateTestUsers):\n def setUp(self):\n self.user = self.create_user(\"test-user\")\n\n def test_ConcordiaLoginView(self):\n with patch(\"concordia.turnstile.fields.TurnstileField.validate\") as mock:\n mock.side_effect = forms.ValidationError(\n \"Testing error\", code=\"invalid_turnstile\"\n )\n response = self.client.post(\n reverse(\"registration_login\"),\n data={\"username\": self.user.username, \"password\": self.user._password},\n )\n self.assertIn(\"user\", response.context)\n self.assertFalse(response.context[\"user\"].is_authenticated)\n\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n response = self.client.post(\n reverse(\"registration_login\"),\n data={\"username\": self.user.username, \"password\": self.user._password},\n follow=True,\n )\n self.assertRedirects(\n response,\n expected_url=reverse(\"homepage\"),\n target_status_code=200,\n )\n self.assertIn(\"user\", response.context)\n self.assertTrue(response.context[\"user\"].is_authenticated)\n\n\nclass TranscriptionViewTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.asset = create_asset()\n\n def test_rollback_transcription(self):\n path = reverse(\"rollback-transcription\", args=[self.asset.id])\n self.login_user()\n\n # Test rollback when there are no transcriptions\n response = self.client.post(path)\n self.assertEqual(400, response.status_code)\n self.assertIn(\"error\", response.json())\n\n transcription1 = create_transcription(\n asset=self.asset, text=\"Test transcription 1\"\n )\n user = transcription1.user\n\n # Test rollback when there are no transcriptions to rollback to\n response = self.client.post(path)\n self.assertEqual(400, response.status_code)\n self.assertIn(\"error\", response.json())\n\n # Test successful rollback\n create_transcription(asset=self.asset, user=user, text=\"Test transcription 2\")\n response = self.client.post(path)\n self.assertEqual(201, response.status_code)\n response_json = response.json()\n self.assertIn(\"id\", response_json)\n self.assertIn(\"text\", response_json)\n self.assertEqual(response_json[\"text\"], transcription1.text)\n self.assertIn(\"undo_available\", response_json)\n self.assertEqual(response_json[\"undo_available\"], False)\n self.assertIn(\"redo_available\", response_json)\n self.assertEqual(response_json[\"redo_available\"], True)\n\n # Test after a rollforward\n self.asset.rollforward_transcription(user)\n response = self.client.post(path)\n self.assertEqual(201, response.status_code)\n response_json = response.json()\n self.assertIn(\"id\", response_json)\n self.assertIn(\"text\", response_json)\n self.assertEqual(response_json[\"text\"], transcription1.text)\n self.assertIn(\"undo_available\", response_json)\n self.assertEqual(response_json[\"undo_available\"], False)\n self.assertIn(\"redo_available\", response_json)\n self.assertEqual(response_json[\"redo_available\"], True)\n\n # Test anonymous user\n self.client.logout()\n create_transcription(asset=self.asset, user=user, text=\"Test transcription 3\")\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n response = self.client.post(path)\n self.assertEqual(201, response.status_code)\n response_json = response.json()\n self.assertIn(\"id\", response_json)\n self.assertIn(\"text\", response_json)\n self.assertEqual(response_json[\"text\"], transcription1.text)\n self.assertIn(\"undo_available\", response_json)\n self.assertEqual(response_json[\"undo_available\"], False)\n self.assertIn(\"redo_available\", response_json)\n self.assertEqual(response_json[\"redo_available\"], True)\n\n def test_rollforward_transcription(self):\n path = reverse(\"rollforward-transcription\", args=[self.asset.id])\n self.login_user()\n\n # Test rollforward when there are no transcriptions\n response = self.client.post(path)\n self.assertEqual(400, response.status_code)\n self.assertIn(\"error\", response.json())\n\n transcription1 = create_transcription(\n asset=self.asset, text=\"Test transcription 1\"\n )\n user = transcription1.user\n\n # Test rollback when there are no transcriptions to rollforward to\n response = self.client.post(path)\n self.assertEqual(400, response.status_code)\n self.assertIn(\"error\", response.json())\n\n # Test successful rollforward, which requires a rollback first\n transcription2 = create_transcription(\n asset=self.asset, user=user, text=\"Test transcription 2\"\n )\n self.asset.rollback_transcription(user)\n response = self.client.post(path)\n self.assertEqual(201, response.status_code)\n response_json = response.json()\n self.assertIn(\"id\", response_json)\n self.assertIn(\"text\", response_json)\n self.assertEqual(response_json[\"text\"], transcription2.text)\n self.assertIn(\"undo_available\", response_json)\n self.assertEqual(response_json[\"undo_available\"], True)\n self.assertIn(\"redo_available\", response_json)\n self.assertEqual(response_json[\"redo_available\"], False)\n\n # Test aftering rolling back then creating a new transcription\n self.asset.rollback_transcription(user)\n create_transcription(asset=self.asset, user=user, text=\"Test transcription 3\")\n response = self.client.post(path)\n self.assertEqual(400, response.status_code)\n self.assertIn(\"error\", response.json())\n\n # Test anonymous user after a rollback\n self.client.logout()\n transcription3 = create_transcription(\n asset=self.asset, user=user, text=\"Test transcription 3\"\n )\n self.asset.rollback_transcription(user)\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n response = self.client.post(path)\n response_json = response.json()\n self.assertEqual(201, response.status_code)\n self.assertIn(\"id\", response_json)\n self.assertIn(\"text\", response_json)\n self.assertEqual(response_json[\"text\"], transcription3.text)\n self.assertIn(\"undo_available\", response_json)\n self.assertEqual(response_json[\"undo_available\"], True)\n self.assertIn(\"redo_available\", response_json)\n self.assertEqual(response_json[\"redo_available\"], False)\n\n def tearDown(self):\n post_save.connect(on_transcription_save, sender=Transcription)\n\n\n@override_settings(\n CACHES={\n \"visualization_cache\": {\n \"BACKEND\": \"django.core.cache.backends.locmem.LocMemCache\",\n }\n }\n)\nclass VisualizationDataViewTests(TestCase):\n def setUp(self):\n self.factory = RequestFactory()\n self.cache = caches[\"visualization_cache\"]\n VisualizationDataView.cache = self.cache\n self.cache.clear()\n self.view = VisualizationDataView.as_view()\n\n def test_get_missing_data_returns_404(self):\n # If no entry exists in the cache under the given name,\n # the view should return a 404 with a JSON error message.\n request = self.factory.get(\"/visualizations/data/missing-key/\")\n response = self.view(request, name=\"missing-key\")\n self.assertEqual(response.status_code, 404)\n self.assertEqual(response[\"Content-Type\"], \"application/json\")\n data = json.loads(response.content)\n self.assertEqual(\n data, {\"error\": \"No visualization data found for 'missing-key'\"}\n )\n\n def test_get_existing_data_returns_200_and_json(self):\n # When the cache contains data for the given name,\n # the view should return it as JSON with status 200.\n sample_data = {\"foo\": \"bar\", \"numbers\": [1, 2, 3]}\n self.cache.set(\"sample-key\", sample_data)\n request = self.factory.get(\"/visualizations/data/sample-key/\")\n response = self.view(request, name=\"sample-key\")\n self.assertEqual(response.status_code, 200)\n self.assertEqual(response[\"Content-Type\"], \"application/json\")\n data = json.loads(response.content)\n self.assertEqual(data, sample_data)\n"
},
{
"path": "concordia/tests/test_views_asset_reservation.py",
"content": "from datetime import timedelta\n\nfrom django.conf import settings\nfrom django.contrib.auth.models import User\nfrom django.db.models.signals import post_save\nfrom django.test import (\n RequestFactory,\n TransactionTestCase,\n override_settings,\n)\nfrom django.urls import reverse\nfrom django.utils.timezone import now\n\nfrom concordia.models import (\n AssetTranscriptionReservation,\n Transcription,\n)\nfrom concordia.signals.handlers import on_transcription_save\nfrom concordia.tasks.reservations import (\n delete_old_tombstoned_reservations,\n expire_inactive_asset_reservations,\n tombstone_old_active_asset_reservations,\n)\nfrom concordia.utils import get_anonymous_user, get_or_create_reservation_token\n\nfrom .utils import (\n CreateTestUsers,\n JSONAssertMixin,\n create_asset,\n)\n\n\n@override_settings(\n RATELIMIT_ENABLE=False, SESSION_ENGINE=\"django.contrib.sessions.backends.cache\"\n)\nclass AssetReservationViewTests(CreateTestUsers, JSONAssertMixin, TransactionTestCase):\n def test_asset_reservation(self):\n \"\"\"\n Test the basic Asset reservation process\n \"\"\"\n\n self.login_user()\n self._asset_reservation_test_payload(self.user.pk)\n\n def test_asset_reservation_anonymously(self):\n \"\"\"\n Test the basic Asset reservation process as an anonymous user\n \"\"\"\n\n anon_user = get_anonymous_user()\n self._asset_reservation_test_payload(anon_user.pk, anonymous=True)\n\n def _asset_reservation_test_payload(self, user_id, anonymous=False):\n asset = create_asset()\n\n # Acquire the reservation: 1 acquire\n # + 1 reservation check\n # + 1 logging if not anonymous\n # + 1 session if not anonymous and using a database session engine:\n expected_update_queries = 2\n if not anonymous:\n expected_update_queries += 1 # Added by django-structlog middleware\n if settings.SESSION_ENGINE.endswith(\"db\"):\n expected_update_queries += 1 # Added by database session engine\n # We don't need to add an extra query for accessing request.user\n # because the django-structlog middleware will do that for non-anonymous\n expected_acquire_queries = expected_update_queries\n else:\n expected_acquire_queries = expected_update_queries + 1\n\n with self.assertNumQueries(expected_acquire_queries):\n resp = self.client.post(reverse(\"reserve-asset\", args=(asset.pk,)))\n data = self.assertValidJSON(resp, expected_status=200)\n\n reservation = AssetTranscriptionReservation.objects.get()\n self.assertEqual(reservation.reservation_token, data[\"reservation_token\"])\n self.assertEqual(reservation.asset, asset)\n\n # Confirm that an update did not change the pk when it updated the timestamp:\n\n with self.assertNumQueries(expected_update_queries):\n resp = self.client.post(reverse(\"reserve-asset\", args=(asset.pk,)))\n data = self.assertValidJSON(resp, expected_status=200)\n self.assertEqual(1, AssetTranscriptionReservation.objects.count())\n updated_reservation = AssetTranscriptionReservation.objects.get()\n self.assertEqual(\n updated_reservation.reservation_token, data[\"reservation_token\"]\n )\n self.assertEqual(updated_reservation.asset, asset)\n self.assertEqual(reservation.created_on, updated_reservation.created_on)\n self.assertLess(reservation.created_on, updated_reservation.updated_on)\n\n # Release the reservation now that we're done:\n # 1 release\n # + 1 logging if not anonymous\n # + 1 session if not anonymous and using a database\n expected_release_queries = 1\n if not anonymous:\n expected_release_queries += 1 # Added by django-structlog middleware\n if settings.SESSION_ENGINE.endswith(\"db\"):\n expected_release_queries += 1\n\n with self.assertNumQueries(expected_release_queries):\n resp = self.client.post(\n reverse(\"reserve-asset\", args=(asset.pk,)), data={\"release\": True}\n )\n data = self.assertValidJSON(resp, expected_status=200)\n self.assertEqual(\n updated_reservation.reservation_token, data[\"reservation_token\"]\n )\n\n self.assertEqual(0, AssetTranscriptionReservation.objects.count())\n\n def test_asset_reservation_competition(self):\n \"\"\"\n Confirm that two users cannot reserve the same asset at the same time\n \"\"\"\n\n asset = create_asset()\n\n # We'll reserve the test asset as the anonymous user and then attempt\n # to edit it after logging in\n\n # 4 queries =\n # 1 expiry + 1 acquire + 2 get user ID + 2 get user profile from request\n with self.assertNumQueries(6):\n resp = self.client.post(reverse(\"reserve-asset\", args=(asset.pk,)))\n self.assertEqual(200, resp.status_code)\n self.assertEqual(1, AssetTranscriptionReservation.objects.count())\n\n # Clear the login session so the reservation_token will be regenerated:\n self.client.logout()\n self.login_user()\n\n # 1 session check + 1 acquire + get user ID from request\n with self.assertNumQueries(3 if settings.SESSION_ENGINE.endswith(\"db\") else 2):\n resp = self.client.post(reverse(\"reserve-asset\", args=(asset.pk,)))\n self.assertEqual(409, resp.status_code)\n self.assertEqual(1, AssetTranscriptionReservation.objects.count())\n\n def test_asset_reservation_expiration(self):\n \"\"\"\n Simulate an expired reservation which should not cause the request to fail\n \"\"\"\n asset = create_asset()\n\n stale_reservation = AssetTranscriptionReservation( # nosec\n asset=asset, reservation_token=\"stale\"\n )\n stale_reservation.full_clean()\n stale_reservation.save()\n # Backdate the object as if it happened 31 minutes ago:\n old_timestamp = now() - timedelta(minutes=31)\n AssetTranscriptionReservation.objects.update(\n created_on=old_timestamp, updated_on=old_timestamp\n )\n\n expire_inactive_asset_reservations()\n\n self.login_user()\n\n # 1 reservation check + 1 acquire + 1 get user ID from request\n expected_queries = 3\n if settings.SESSION_ENGINE.endswith(\"db\"):\n # 1 session check\n expected_queries += 1\n\n with self.assertNumQueries(expected_queries):\n resp = self.client.post(reverse(\"reserve-asset\", args=(asset.pk,)))\n\n data = self.assertValidJSON(resp, expected_status=200)\n self.assertEqual(1, AssetTranscriptionReservation.objects.count())\n reservation = AssetTranscriptionReservation.objects.get()\n self.assertEqual(reservation.reservation_token, data[\"reservation_token\"])\n\n def test_asset_reservation_tombstone(self):\n \"\"\"\n Simulate a tombstoned reservation which should:\n - return 408 during the tombstone period\n - during the tombstone period, another user may\n obtain the reservation but the original user may not\n \"\"\"\n asset = create_asset()\n self.login_user()\n request_factory = RequestFactory()\n request = request_factory.get(\"/\")\n request.session = {}\n reservation_token = get_or_create_reservation_token(request)\n\n session = self.client.session\n session[\"reservation_token\"] = reservation_token\n session.save()\n\n tombstone_reservation = AssetTranscriptionReservation( # nosec\n asset=asset, reservation_token=reservation_token\n )\n tombstone_reservation.full_clean()\n tombstone_reservation.save()\n # Backdate the object as if it was created hours ago,\n # even if it was recently updated\n old_timestamp = now() - timedelta(\n hours=settings.TRANSCRIPTION_RESERVATION_TOMBSTONE_HOURS + 1\n )\n current_timestamp = now()\n AssetTranscriptionReservation.objects.update(\n created_on=old_timestamp, updated_on=current_timestamp\n )\n\n tombstone_old_active_asset_reservations()\n self.assertEqual(1, AssetTranscriptionReservation.objects.count())\n reservation = AssetTranscriptionReservation.objects.get()\n self.assertEqual(reservation.tombstoned, True)\n\n # 1 session check + 1 reservation check + 1 logging\n if settings.SESSION_ENGINE.endswith(\"db\"):\n expected_queries = 3\n else:\n expected_queries = 2\n\n with self.assertNumQueries(expected_queries):\n resp = self.client.post(reverse(\"reserve-asset\", args=(asset.pk,)))\n\n self.assertEqual(resp.status_code, 408)\n self.assertEqual(1, AssetTranscriptionReservation.objects.count())\n reservation = AssetTranscriptionReservation.objects.get()\n self.assertEqual(reservation.reservation_token, reservation_token)\n\n self.client.logout()\n\n # 1 reservation check + 1 acquire + 1 get user ID\n expected_queries = 3\n if settings.SESSION_ENGINE.endswith(\"db\"):\n # + 1 session check\n expected_queries += 1\n\n User.objects.create_user(username=\"anonymous\")\n with self.assertNumQueries(expected_queries):\n resp = self.client.post(reverse(\"reserve-asset\", args=(asset.pk,)))\n\n self.assertValidJSON(resp, expected_status=200)\n self.assertEqual(2, AssetTranscriptionReservation.objects.count())\n\n def test_asset_reservation_tombstone_expiration(self):\n \"\"\"\n Simulate a tombstoned reservation which should expire after\n the configured period of time, allowing the original user\n to reserve the asset again\n \"\"\"\n asset = create_asset()\n self.login_user()\n request_factory = RequestFactory()\n request = request_factory.get(\"/\")\n request.session = {}\n reservation_token = get_or_create_reservation_token(request)\n\n session = self.client.session\n session[\"reservation_token\"] = reservation_token\n session.save()\n\n tombstone_reservation = AssetTranscriptionReservation( # nosec\n asset=asset, reservation_token=reservation_token\n )\n tombstone_reservation.full_clean()\n tombstone_reservation.save()\n # Backdate the object as if it was created hours ago\n # and tombstoned hours ago\n old_timestamp = now() - timedelta(\n hours=settings.TRANSCRIPTION_RESERVATION_TOMBSTONE_HOURS\n + settings.TRANSCRIPTION_RESERVATION_TOMBSTONE_LENGTH_HOURS\n + 1\n )\n not_as_old_timestamp = now() - timedelta(\n hours=settings.TRANSCRIPTION_RESERVATION_TOMBSTONE_LENGTH_HOURS + 1\n )\n AssetTranscriptionReservation.objects.update(\n created_on=old_timestamp, updated_on=not_as_old_timestamp, tombstoned=True\n )\n\n delete_old_tombstoned_reservations()\n self.assertEqual(0, AssetTranscriptionReservation.objects.count())\n\n # 1 session check + 1 reservation check + 1 acquire + 1logging\n if settings.SESSION_ENGINE.endswith(\"db\"):\n expected_queries = 4\n else:\n expected_queries = 3\n\n with self.assertNumQueries(expected_queries):\n resp = self.client.post(reverse(\"reserve-asset\", args=(asset.pk,)))\n\n data = self.assertValidJSON(resp, expected_status=200)\n self.assertEqual(1, AssetTranscriptionReservation.objects.count())\n reservation = AssetTranscriptionReservation.objects.get()\n self.assertEqual(reservation.reservation_token, data[\"reservation_token\"])\n self.assertEqual(reservation.tombstoned, False)\n\n def tearDown(self):\n # We'll test the signal handler separately\n post_save.connect(on_transcription_save, sender=Transcription)\n"
},
{
"path": "concordia/tests/test_views_redirect_next_reviewable.py",
"content": "from unittest.mock import patch\n\nfrom django.db.models.signals import post_save\nfrom django.test import (\n TransactionTestCase,\n override_settings,\n)\nfrom django.urls import reverse\nfrom django.utils.timezone import now\n\nfrom concordia.models import (\n Transcription,\n)\nfrom concordia.signals.handlers import on_transcription_save\nfrom concordia.utils import get_anonymous_user\n\nfrom .utils import (\n CreateTestUsers,\n JSONAssertMixin,\n create_asset,\n create_campaign,\n create_item,\n create_project,\n create_topic,\n)\n\n\n@override_settings(\n RATELIMIT_ENABLE=False, SESSION_ENGINE=\"django.contrib.sessions.backends.cache\"\n)\nclass NextReviewableRedirectViewTests(\n CreateTestUsers, JSONAssertMixin, TransactionTestCase\n):\n def test_find_next_reviewable_no_campaign(self):\n user = self.create_user(\"test-user\")\n anon = get_anonymous_user()\n\n # Test case where there are no reviewable assets\n response = self.client.get(reverse(\"redirect-to-next-reviewable-asset\"))\n self.assertRedirects(response, expected_url=\"/\")\n\n asset1 = create_asset(slug=\"test-asset-1\", title=\"Test Asset 1\")\n asset2 = create_asset(\n item=asset1.item, slug=\"test-asset-2\", title=\"Test Asset 2\"\n )\n asset3 = create_asset(\n item=asset1.item, slug=\"test-asset-3\", title=\"Test Asset 3\"\n )\n campaign = asset1.item.project.campaign\n\n t1 = Transcription(asset=asset1, user=user, text=\"test\", submitted=now())\n t1.full_clean()\n t1.save()\n\n t2 = Transcription(asset=asset2, user=anon, text=\"test\", submitted=now())\n t2.full_clean()\n t2.save()\n\n t3 = Transcription(asset=asset3, user=anon, text=\"test\", submitted=now())\n t3.full_clean()\n t3.save()\n\n response = self.client.get(reverse(\"redirect-to-next-reviewable-asset\"))\n self.assertRedirects(response, expected_url=asset1.get_absolute_url())\n\n # Test logged in user (this creates a new user)\n # asset1 is no longer available due to the request above reserving it\n self.login_user()\n response = self.client.get(reverse(\"redirect-to-next-reviewable-asset\"))\n self.assertRedirects(response, expected_url=asset2.get_absolute_url())\n\n # Configure campaign to be next review cmpaign for tests below\n campaign.next_review_campaign = True\n campaign.save()\n\n # Test when next reviewable campaign doesn't exist and there\n # are no other campaigns/assets\n with patch(\"concordia.models.Campaign.objects.get\") as mock:\n mock.side_effect = IndexError\n response = self.client.get(reverse(\"redirect-to-next-reviewable-asset\"))\n self.assertRedirects(response, expected_url=\"/\")\n\n # Test case when a campaign is configured to be default next reviewable\n response = self.client.get(reverse(\"redirect-to-next-reviewable-asset\"))\n self.assertRedirects(response, expected_url=asset3.get_absolute_url())\n\n # Test when next reviewable campaign has no reviewable assets\n asset1.delete()\n asset2.delete()\n response = self.client.get(reverse(\"redirect-to-next-reviewable-asset\"))\n self.assertRedirects(response, expected_url=\"/\")\n\n # Test when next reviewable campaign has no reviewable assets\n # and other campaigns exist and have no reviewable assets\n create_campaign(slug=\"test-campaign-2\")\n response = self.client.get(reverse(\"redirect-to-next-reviewable-asset\"))\n self.assertRedirects(response, expected_url=\"/\")\n\n def test_find_next_reviewable_campaign(self):\n anon = get_anonymous_user()\n\n asset1 = create_asset(slug=\"test-review-asset-1\", title=\"Test Asset 1\")\n asset2 = create_asset(\n item=asset1.item, slug=\"test-review-asset-2\", title=\"Test Asset 2\"\n )\n\n t1 = Transcription(asset=asset1, user=anon, text=\"test\", submitted=now())\n t1.full_clean()\n t1.save()\n\n t2 = Transcription(asset=asset2, user=anon, text=\"test\", submitted=now())\n t2.full_clean()\n t2.save()\n\n campaign = asset1.item.project.campaign\n\n # Anonymous user test\n response = self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-reviewable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n )\n )\n self.assertRedirects(response, expected_url=asset1.get_absolute_url())\n\n # Authenticated user test\n # asset1 is no longer available since the previous request reserved it\n self.login_user()\n response = self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-reviewable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n )\n )\n self.assertRedirects(response, expected_url=asset2.get_absolute_url())\n\n def test_find_next_reviewable_topic(self):\n anon = get_anonymous_user()\n\n asset1 = create_asset(slug=\"test-review-asset-1\")\n asset2 = create_asset(item=asset1.item, slug=\"test-review-asset-2\")\n project = asset1.item.project\n topic = create_topic(project=project)\n\n t1 = Transcription(asset=asset1, user=anon, text=\"test\", submitted=now())\n t1.full_clean()\n t1.save()\n\n t2 = Transcription(asset=asset2, user=anon, text=\"test\", submitted=now())\n t2.full_clean()\n t2.save()\n\n # Anonymous user test\n response = self.client.get(\n reverse(\n \"redirect-to-next-reviewable-topic-asset\",\n kwargs={\"topic_slug\": topic.slug},\n )\n )\n self.assertRedirects(response, expected_url=asset1.get_absolute_url())\n\n # Authenticated user test\n # We expect that asset1 is no longer available. Even though\n # anonymous users can't reserve assets for review, we still will\n # have removed the asset from the NextReviewableTopicAsset table\n # to ensure two users don't receive the same asset\n self.login_user()\n response = self.client.get(\n reverse(\n \"redirect-to-next-reviewable-topic-asset\",\n kwargs={\"topic_slug\": topic.slug},\n )\n )\n self.assertRedirects(response, expected_url=asset2.get_absolute_url())\n\n def test_find_next_reviewable_unlisted_campaign(self):\n anon = get_anonymous_user()\n\n unlisted_campaign = create_campaign(\n slug=\"campaign-transcribe-redirect-unlisted\",\n title=\"Test Unlisted Review Redirect Campaign\",\n unlisted=True,\n )\n unlisted_project = create_project(\n title=\"Unlisted Project\",\n slug=\"unlisted-project\",\n campaign=unlisted_campaign,\n )\n unlisted_item = create_item(\n title=\"Unlisted Item\",\n item_id=\"unlisted-item\",\n item_url=\"https://blah.com/unlisted-item\",\n project=unlisted_project,\n )\n\n asset1 = create_asset(slug=\"test-asset-1\", item=unlisted_item)\n asset2 = create_asset(item=asset1.item, slug=\"test-asset-2\")\n\n t1 = Transcription(asset=asset1, user=anon, text=\"test\", submitted=now())\n t1.full_clean()\n t1.save()\n\n t2 = Transcription(asset=asset2, user=anon, text=\"test\", submitted=now())\n t2.full_clean()\n t2.save()\n\n response = self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-reviewable-campaign-asset\",\n kwargs={\"campaign_slug\": unlisted_campaign.slug},\n )\n )\n\n self.assertRedirects(response, expected_url=asset1.get_absolute_url())\n\n def tearDown(self):\n # We'll test the signal handler separately\n post_save.connect(on_transcription_save, sender=Transcription)\n"
},
{
"path": "concordia/tests/test_views_redirect_next_transcribable.py",
"content": "from unittest.mock import patch\n\nfrom django.db.models.signals import post_save\nfrom django.test import (\n TransactionTestCase,\n override_settings,\n)\nfrom django.urls import reverse\n\nfrom concordia.models import (\n AssetTranscriptionReservation,\n Transcription,\n TranscriptionStatus,\n)\nfrom concordia.signals.handlers import on_transcription_save\n\nfrom .utils import (\n CreateTestUsers,\n JSONAssertMixin,\n create_asset,\n create_campaign,\n create_item,\n create_project,\n create_topic,\n)\n\n\n@override_settings(\n RATELIMIT_ENABLE=False, SESSION_ENGINE=\"django.contrib.sessions.backends.cache\"\n)\nclass NextTranscribableRedirectViewTests(\n CreateTestUsers, JSONAssertMixin, TransactionTestCase\n):\n def test_find_next_transcribable_no_campaign(self):\n # Test case where there are no transcribable assets\n resp = self.client.get(reverse(\"redirect-to-next-transcribable-asset\"))\n self.assertRedirects(resp, expected_url=\"/\")\n\n asset1 = create_asset(slug=\"test-asset-1\")\n asset2 = create_asset(item=asset1.item, slug=\"test-asset-2\")\n campaign = asset1.item.project.campaign\n\n resp = self.client.get(reverse(\"redirect-to-next-transcribable-asset\"))\n self.assertRedirects(resp, expected_url=asset1.get_absolute_url())\n\n # Configure next transcription campaign for tests below\n campaign.next_transcription_campaign = True\n campaign.save()\n\n # Test when next transcribable campaign doesn't exist and there\n # are no other campaigns/assets\n with patch(\"concordia.models.Campaign.objects.get\") as mock:\n mock.side_effect = IndexError\n response = self.client.get(reverse(\"redirect-to-next-transcribable-asset\"))\n self.assertRedirects(response, expected_url=\"/\")\n\n # Test case when a campaign is configured to be default next transcribable\n response = self.client.get(reverse(\"redirect-to-next-transcribable-asset\"))\n self.assertRedirects(response, expected_url=asset2.get_absolute_url())\n\n # Test when next transcribable campaign has not transcribable assets\n asset1.delete()\n asset2.delete()\n response = self.client.get(reverse(\"redirect-to-next-transcribable-asset\"))\n self.assertRedirects(response, expected_url=\"/\")\n\n # Test when next transcription campaign has no transcribable assets\n # and other campaigns exist and have no transcribable assets\n create_campaign(slug=\"test-campaign-2\")\n response = self.client.get(reverse(\"redirect-to-next-transcribable-asset\"))\n self.assertRedirects(response, expected_url=\"/\")\n\n def test_find_next_transcribable_campaign(self):\n asset1 = create_asset(slug=\"test-asset-1\")\n asset2 = create_asset(item=asset1.item, slug=\"test-asset-2\")\n campaign = asset1.item.project.campaign\n\n # Anonymous user test\n resp = self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n )\n )\n self.assertRedirects(resp, expected_url=asset1.get_absolute_url())\n\n # Authenticated user test\n self.login_user()\n resp = self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n )\n )\n self.assertRedirects(resp, expected_url=asset2.get_absolute_url())\n\n def test_find_next_transcribable_topic(self):\n asset1 = create_asset(slug=\"test-asset-1\")\n asset2 = create_asset(item=asset1.item, slug=\"test-asset-2\")\n project = asset1.item.project\n topic = create_topic(project=project)\n\n # Anonymous user test\n resp = self.client.get(\n reverse(\n \"redirect-to-next-transcribable-topic-asset\",\n kwargs={\"topic_slug\": topic.slug},\n )\n )\n self.assertRedirects(resp, expected_url=asset1.get_absolute_url())\n\n # Authenticated user test\n self.login_user()\n resp = self.client.get(\n reverse(\n \"redirect-to-next-transcribable-topic-asset\",\n kwargs={\"topic_slug\": topic.slug},\n )\n )\n self.assertRedirects(resp, expected_url=asset2.get_absolute_url())\n\n def test_find_next_transcribable_unlisted_campaign(self):\n unlisted_campaign = create_campaign(\n slug=\"campaign-transcribe-redirect-unlisted\",\n title=\"Test Unlisted Transcribe Redirect Campaign\",\n unlisted=True,\n )\n unlisted_project = create_project(\n title=\"Unlisted Project\",\n slug=\"unlisted-project\",\n campaign=unlisted_campaign,\n )\n unlisted_item = create_item(\n title=\"Unlisted Item\",\n item_id=\"unlisted-item\",\n item_url=\"https://blah.com/unlisted-item\",\n project=unlisted_project,\n )\n\n asset1 = create_asset(slug=\"test-asset-1\", item=unlisted_item)\n create_asset(item=asset1.item, slug=\"test-asset-2\")\n\n response = self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": unlisted_campaign.slug},\n )\n )\n\n self.assertRedirects(response, expected_url=asset1.get_absolute_url())\n\n def test_find_next_transcribable_single_asset(self):\n asset = create_asset()\n campaign = asset.item.project.campaign\n\n resp = self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n )\n )\n\n self.assertRedirects(resp, expected_url=asset.get_absolute_url())\n\n def test_find_next_transcribable_in_singleton_campaign(self):\n asset = create_asset(transcription_status=TranscriptionStatus.SUBMITTED)\n campaign = asset.item.project.campaign\n\n resp = self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n )\n )\n\n self.assertRedirects(resp, expected_url=reverse(\"homepage\"))\n\n def test_find_next_transcribable_project_redirect(self):\n asset = create_asset(transcription_status=TranscriptionStatus.SUBMITTED)\n project = asset.item.project\n campaign = project.campaign\n\n resp = self.client.get(\n \"%s?project=%s\"\n % (\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n ),\n project.slug,\n )\n )\n\n self.assertRedirects(resp, expected_url=reverse(\"homepage\"))\n\n def test_find_next_transcribable_hierarchy(self):\n \"\"\"Confirm that find-next-page selects assets in the expected order\"\"\"\n\n asset = create_asset()\n item = asset.item\n project = item.project\n campaign = project.campaign\n\n asset_in_item = create_asset(item=item, slug=\"test-asset-in-same-item\")\n in_progress_asset_in_item = create_asset(\n item=item,\n slug=\"inprogress-asset-in-same-item\",\n transcription_status=TranscriptionStatus.IN_PROGRESS,\n )\n\n asset_in_project = create_asset(\n item=create_item(project=project, item_id=\"other-item-in-same-project\"),\n title=\"test-asset-in-same-project\",\n )\n\n asset_in_campaign = create_asset(\n item=create_item(\n project=create_project(campaign=campaign, title=\"other project\"),\n item_id=\"another-item-in-different-project\",\n title=\"item in other project\",\n ),\n slug=\"test-asset-in-same-campaign\",\n )\n\n # Now that we have test assets we'll see what find-next-page gives us as\n # successive test records are marked as submitted and thus ineligible.\n # The expected ordering is that it will favor moving forward (i.e. not\n # landing you on the same asset unless that's the only one available),\n # and will keep you closer to the asset you started from (i.e. within\n # the same item or project in that order).\n\n self.assertRedirects(\n self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n ),\n {\"project\": project.slug, \"item\": item.item_id, \"asset\": asset.pk},\n ),\n asset_in_item.get_absolute_url(),\n )\n\n asset_in_item.transcription_status = TranscriptionStatus.SUBMITTED\n asset_in_item.save()\n AssetTranscriptionReservation.objects.all().delete()\n\n self.assertRedirects(\n self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n ),\n {\"project\": project.slug, \"item\": item.item_id, \"asset\": asset.pk},\n ),\n asset_in_project.get_absolute_url(),\n )\n\n asset_in_project.transcription_status = TranscriptionStatus.SUBMITTED\n asset_in_project.save()\n AssetTranscriptionReservation.objects.all().delete()\n\n self.assertRedirects(\n self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n ),\n {\"project\": project.slug, \"item\": item.item_id, \"asset\": asset.pk},\n ),\n asset_in_campaign.get_absolute_url(),\n )\n\n asset_in_campaign.transcription_status = TranscriptionStatus.SUBMITTED\n asset_in_campaign.save()\n AssetTranscriptionReservation.objects.all().delete()\n\n self.assertRedirects(\n self.client.get(\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": campaign.slug},\n ),\n {\"project\": project.slug, \"item\": item.item_id, \"asset\": asset.pk},\n ),\n in_progress_asset_in_item.get_absolute_url(),\n )\n\n def tearDown(self):\n # We'll test the signal handler separately\n post_save.connect(on_transcription_save, sender=Transcription)\n"
},
{
"path": "concordia/tests/test_views_tags.py",
"content": "from unittest.mock import patch\n\nfrom django import forms\nfrom django.contrib.auth.models import User\nfrom django.db.models.signals import post_save\nfrom django.test import (\n TransactionTestCase,\n override_settings,\n)\nfrom django.urls import reverse\n\nfrom concordia.models import (\n Transcription,\n)\nfrom concordia.signals.handlers import on_transcription_save\n\nfrom .utils import (\n CreateTestUsers,\n JSONAssertMixin,\n create_asset,\n)\n\n\n@override_settings(\n RATELIMIT_ENABLE=False, SESSION_ENGINE=\"django.contrib.sessions.backends.cache\"\n)\nclass TagSubmissionViewTests(CreateTestUsers, JSONAssertMixin, TransactionTestCase):\n def test_anonymous_tag_submission(self):\n \"\"\"Confirm that anonymous users cannot submit tags\"\"\"\n asset = create_asset()\n submit_url = reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk})\n\n resp = self.client.post(submit_url, data={\"tags\": [\"foo\", \"bar\"]})\n self.assertRedirects(resp, \"%s?next=%s\" % (reverse(\"login\"), submit_url))\n\n def test_tag_submission(self):\n asset = create_asset()\n\n self.login_user()\n\n test_tags = [\"foo\", \"bar\"]\n\n resp = self.client.post(\n reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk}),\n data={\"tags\": test_tags},\n )\n data = self.assertValidJSON(resp, expected_status=200)\n self.assertIn(\"user_tags\", data)\n self.assertIn(\"all_tags\", data)\n\n self.assertEqual(sorted(test_tags), data[\"user_tags\"])\n self.assertEqual(sorted(test_tags), data[\"all_tags\"])\n\n def test_invalid_tag_submission(self):\n asset = create_asset()\n\n self.login_user()\n\n test_tags = [\"foo\", \"bar\"]\n\n with patch(\"concordia.models.Tag.full_clean\") as mock:\n mock.side_effect = forms.ValidationError(\"Testing error\")\n resp = self.client.post(\n reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk}),\n data={\"tags\": test_tags},\n )\n data = self.assertValidJSON(resp, expected_status=400)\n self.assertIn(\"error\", data)\n\n def test_tag_submission_with_diacritics(self):\n asset = create_asset()\n\n self.login_user()\n\n test_tags = [\"Café\", \"chÒteau\", \"señor\", \"façade\"]\n\n resp = self.client.post(\n reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk}),\n data={\"tags\": test_tags},\n )\n data = self.assertValidJSON(resp, expected_status=200)\n self.assertIn(\"user_tags\", data)\n self.assertIn(\"all_tags\", data)\n\n self.assertEqual(sorted(test_tags), data[\"user_tags\"])\n self.assertEqual(sorted(test_tags), data[\"all_tags\"])\n\n def test_tag_submission_with_multiple_users(self):\n asset = create_asset()\n self.login_user()\n\n test_tags = [\"foo\", \"bar\"]\n\n resp = self.client.post(\n reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk}),\n data={\"tags\": test_tags},\n )\n data = self.assertValidJSON(resp, expected_status=200)\n self.assertIn(\"user_tags\", data)\n self.assertIn(\"all_tags\", data)\n\n self.assertEqual(sorted(test_tags), data[\"user_tags\"])\n self.assertEqual(sorted(test_tags), data[\"all_tags\"])\n\n def test_duplicate_tag_submission(self):\n \"\"\"Confirm that tag values cannot be duplicated\"\"\"\n asset = create_asset()\n\n self.login_user()\n\n resp = self.client.post(\n reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk}),\n data={\"tags\": [\"foo\", \"bar\", \"baaz\"]},\n )\n data = self.assertValidJSON(resp, expected_status=200)\n\n second_user = self.create_test_user(\n username=\"second_tester\", email=\"second_tester@example.com\"\n )\n self.client.login(username=second_user.username, password=second_user._password)\n\n resp = self.client.post(\n reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk}),\n data={\"tags\": [\"foo\", \"bar\", \"quux\"]},\n )\n data = self.assertValidJSON(resp, expected_status=200)\n\n # Even though the user submitted (through some horrible bug) duplicate\n # values, they should not be stored:\n self.assertEqual([\"bar\", \"foo\", \"quux\"], data[\"user_tags\"])\n # Users are allowed to delete other users' tags, so since the second\n # user didn't send the \"baaz\" tag, it was removed\n self.assertEqual([\"bar\", \"foo\", \"quux\"], data[\"all_tags\"])\n\n def test_tag_deletion(self):\n asset = create_asset()\n self.login_user()\n\n initial_tags = [\"foo\", \"bar\"]\n self.client.post(\n reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk}),\n data={\"tags\": initial_tags},\n )\n updated_tags = [\n \"foo\",\n ]\n resp = self.client.post(\n reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk}),\n data={\"tags\": updated_tags},\n )\n data = self.assertValidJSON(resp, expected_status=200)\n self.assertIn(\"user_tags\", data)\n self.assertIn(\"all_tags\", data)\n\n self.assertCountEqual(updated_tags, data[\"user_tags\"])\n self.assertCountEqual(updated_tags, data[\"all_tags\"])\n\n def test_tag_deletion_with_multiple_users(self):\n asset = create_asset()\n self.login_user(\"first_user\")\n initial_tags = [\"foo\", \"bar\"]\n resp = self.client.post(\n reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk}),\n data={\"tags\": initial_tags},\n )\n self.assertIn(\n \"first_user\",\n asset.userassettagcollection_set.values().values_list(\n \"user__username\", flat=True\n ),\n )\n data = self.assertValidJSON(resp, expected_status=200)\n self.assertIn(\"user_tags\", data)\n self.assertIn(\"all_tags\", data)\n self.assertCountEqual(initial_tags, data[\"user_tags\"])\n self.assertCountEqual(initial_tags, data[\"all_tags\"])\n\n self.client.logout()\n\n second_user = self.create_test_user(\"second_user\")\n self.client.login(username=second_user.username, password=second_user._password)\n updated_tags = [\n \"foo\",\n ]\n resp = self.client.post(\n reverse(\"submit-tags\", kwargs={\"asset_pk\": asset.pk}),\n data={\"tags\": updated_tags},\n )\n data = self.assertValidJSON(resp, expected_status=200)\n\n self.assertIn(\n \"second_user\",\n asset.userassettagcollection_set.values().values_list(\n \"user__username\", flat=True\n ),\n )\n self.assertEqual(asset.userassettagcollection_set.count(), 2)\n self.assertEqual(\n User.objects.filter(userassettagcollection__asset=asset).count(), 2\n )\n self.assertIn(\"user_tags\", data)\n self.assertIn(\"all_tags\", data)\n self.assertCountEqual(updated_tags, data[\"user_tags\"])\n\n def tearDown(self):\n # We'll test the signal handler separately\n post_save.connect(on_transcription_save, sender=Transcription)\n"
},
{
"path": "concordia/tests/test_views_topics.py",
"content": "from django.core.cache import caches\nfrom django.test import TestCase, override_settings\nfrom django.urls import reverse\n\nfrom concordia.models import ProjectTopic, TranscriptionStatus\n\nfrom .utils import (\n CreateTestUsers,\n JSONAssertMixin,\n create_asset,\n create_campaign,\n create_item,\n create_project,\n create_topic,\n)\n\n\n@override_settings(\n RATELIMIT_ENABLE=False,\n SESSION_ENGINE=\"django.contrib.sessions.backends.cache\",\n CACHES={\n \"default\": {\"BACKEND\": \"django.core.cache.backends.dummy.DummyCache\"},\n \"view_cache\": {\"BACKEND\": \"django.core.cache.backends.dummy.DummyCache\"},\n },\n)\nclass TopicDetailViewTests(CreateTestUsers, JSONAssertMixin, TestCase):\n \"\"\"\n Focused tests for the Topic detail view.\n \"\"\"\n\n def setUp(self):\n for cache in caches.all():\n cache.clear()\n\n def tearDown(self):\n for cache in caches.all():\n cache.clear()\n\n def test_topic_detail_basic(self):\n topic = create_topic(title=\"GET Topic\", slug=\"get-topic\")\n response = self.client.get(reverse(\"topic-detail\", args=(topic.slug,)))\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/topic_detail.html\"\n )\n self.assertContains(response, topic.title)\n\n def test_unlisted_topic_detail_view(self):\n c2 = create_topic(\n title=\"GET Unlisted Topic\", unlisted=True, slug=\"get-unlisted-topic\"\n )\n\n response2 = self.client.get(reverse(\"topic-detail\", args=(c2.slug,)))\n\n self.assertEqual(response2.status_code, 200)\n self.assertTemplateUsed(\n response2, template_name=\"transcriptions/topic_detail.html\"\n )\n self.assertContains(response2, c2.title)\n\n def test_topic_detail_with_status_sets_querystring(self):\n \"\"\"\n When a valid transcription_status is supplied, sublevel_querystring\n contains only that param.\n \"\"\"\n topic = create_topic(title=\"GET Topic\", slug=\"get-topic\")\n response = self.client.get(\n reverse(\"topic-detail\", args=(topic.slug,)),\n {\"transcription_status\": \"not_started\"},\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(\n response, template_name=\"transcriptions/topic_detail.html\"\n )\n self.assertContains(response, topic.title)\n self.assertIn(\"sublevel_querystring\", response.context)\n self.assertEqual(\n response.context[\"sublevel_querystring\"], \"transcription_status=not_started\"\n )\n\n def test_url_filter_links_without_sublevel_querystring(self):\n \"\"\"\n With a project-level url_filter and no sublevel filter, links for that\n project include transcription_status=, while projects without\n a url_filter do not include a transcription_status param.\n \"\"\"\n topic = create_topic(title=\"Filter Topic\", slug=\"filter-topic\")\n campaign = create_campaign(title=\"Filter Test Campaign\", slug=\"filter-test\")\n\n project_with_filter = create_project(\n campaign=campaign, title=\"Project With Filter\", slug=\"with-filter\"\n )\n project_without_filter = create_project(\n campaign=campaign, title=\"Project Without Filter\", slug=\"without-filter\"\n )\n\n ProjectTopic.objects.create(\n project=project_with_filter,\n topic=topic,\n url_filter=TranscriptionStatus.SUBMITTED,\n )\n ProjectTopic.objects.create(\n project=project_without_filter,\n topic=topic,\n url_filter=None,\n )\n\n response = self.client.get(reverse(\"topic-detail\", args=(topic.slug,)))\n self.assertEqual(response.status_code, 200)\n\n # project_with_filter has ?transcription_status=submitted\n # (appears twice: image+title)\n self.assertContains(\n response,\n f\"/campaigns/{campaign.slug}/{project_with_filter.slug}/?transcription_status=submitted\",\n 2,\n )\n # project_without_filter should not include any transcription_status param\n self.assertNotContains(\n response,\n f\"/campaigns/{campaign.slug}/{project_without_filter.slug}/?transcription_status=\",\n )\n\n def test_sublevel_querystring_only_keeps_transcription_status(self):\n \"\"\"\n If extra params are provided along with transcription_status, only\n transcription_status is retained in sublevel_querystring.\n \"\"\"\n topic = create_topic(title=\"GET Topic\", slug=\"get-topic\")\n response = self.client.get(\n reverse(\"topic-detail\", args=(topic.slug,)),\n {\"transcription_status\": \"not_started\", \"another_param\": \"some_value\"},\n )\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"sublevel_querystring\", response.context)\n self.assertEqual(\n response.context[\"sublevel_querystring\"], \"transcription_status=not_started\"\n )\n\n def test_with_status_and_no_assets_excludes_projects(self):\n \"\"\"\n When a transcription_status is present and projects have no assets,\n those projects are excluded (no links rendered).\n \"\"\"\n topic = create_topic(title=\"Filter Topic\", slug=\"filter-topic\")\n campaign = create_campaign(title=\"Filter Test Campaign\", slug=\"filter-test\")\n\n project_with_filter = create_project(\n campaign=campaign, title=\"Project With Filter\", slug=\"with-filter\"\n )\n project_without_filter = create_project(\n campaign=campaign, title=\"Project Without Filter\", slug=\"without-filter\"\n )\n\n ProjectTopic.objects.create(\n project=project_with_filter,\n topic=topic,\n url_filter=TranscriptionStatus.SUBMITTED,\n )\n ProjectTopic.objects.create(\n project=project_without_filter,\n topic=topic,\n url_filter=None,\n )\n\n response = self.client.get(\n reverse(\"topic-detail\", args=(topic.slug,)),\n {\"transcription_status\": \"not_started\", \"another_param\": \"some_value\"},\n )\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"sublevel_querystring\", response.context)\n self.assertEqual(\n response.context[\"sublevel_querystring\"], \"transcription_status=not_started\"\n )\n\n # No assets exist, so neither project should appear with the filter applied\n self.assertContains(\n response,\n f\"/campaigns/{campaign.slug}/{project_with_filter.slug}/?transcription_status=not_started\",\n 0,\n )\n self.assertContains(\n response,\n f\"/campaigns/{campaign.slug}/{project_without_filter.slug}/?transcription_status=not_started\",\n 0,\n )\n\n def test_with_status_and_assets_uses_sublevel_and_overrides_url_filter(self):\n \"\"\"\n When assets exist and a transcription_status is supplied, projects with no\n url_filter are shown using the sublevel filter. Projects with a url_filter\n that does not match the sublevel filter are excluded.\n \"\"\"\n topic = create_topic(title=\"Filter Topic\", slug=\"filter-topic\")\n campaign = create_campaign(title=\"Filter Test Campaign\", slug=\"filter-test\")\n\n project_with_filter = create_project(\n campaign=campaign, title=\"Project With Filter\", slug=\"with-filter\"\n )\n project_without_filter = create_project(\n campaign=campaign, title=\"Project Without Filter\", slug=\"without-filter\"\n )\n\n ProjectTopic.objects.create(\n project=project_with_filter,\n topic=topic,\n url_filter=TranscriptionStatus.SUBMITTED,\n )\n ProjectTopic.objects.create(\n project=project_without_filter,\n topic=topic,\n url_filter=None,\n )\n\n # Add assets so eligible projects will display\n item_with_filter = create_item(project=project_with_filter)\n create_asset(item=item_with_filter)\n item_without_filter = create_item(project=project_without_filter)\n create_asset(item=item_without_filter)\n\n response = self.client.get(\n reverse(\"topic-detail\", args=(topic.slug,)),\n {\"transcription_status\": \"not_started\", \"another_param\": \"some_value\"},\n )\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"sublevel_querystring\", response.context)\n self.assertEqual(\n response.context[\"sublevel_querystring\"], \"transcription_status=not_started\"\n )\n\n # Project WITH a mismatching url_filter should be excluded\n self.assertContains(\n response,\n f\"/campaigns/{campaign.slug}/{project_with_filter.slug}/?transcription_status=not_started\",\n 0,\n )\n\n # Project WITHOUT a url_filter should use the sublevel filter\n # (appears twice: image + title)\n self.assertContains(\n response,\n f\"/campaigns/{campaign.slug}/{project_without_filter.slug}/?transcription_status=not_started\",\n 2,\n )\n\n def test_with_status_and_assets_includes_matching_url_filter(self):\n \"\"\"\n When assets exist and a transcription_status is supplied, projects with a\n matching url_filter should be included, and links should use that status.\n \"\"\"\n topic = create_topic(title=\"Filter Topic\", slug=\"filter-topic\")\n campaign = create_campaign(title=\"Filter Test Campaign\", slug=\"filter-test\")\n\n project_with_filter = create_project(\n campaign=campaign, title=\"Project With Filter\", slug=\"with-filter\"\n )\n project_without_filter = create_project(\n campaign=campaign, title=\"Project Without Filter\", slug=\"without-filter\"\n )\n\n ProjectTopic.objects.create(\n project=project_with_filter,\n topic=topic,\n url_filter=TranscriptionStatus.SUBMITTED,\n )\n ProjectTopic.objects.create(\n project=project_without_filter,\n topic=topic,\n url_filter=None,\n )\n\n # Ensure both projects have at least one asset counted as \"submitted\"\n item_with_filter = create_item(project=project_with_filter)\n a1 = create_asset(item=item_with_filter)\n a1.transcription_status = TranscriptionStatus.SUBMITTED\n a1.save(update_fields=[\"transcription_status\"])\n\n item_without_filter = create_item(project=project_without_filter)\n a2 = create_asset(item=item_without_filter)\n a2.transcription_status = TranscriptionStatus.SUBMITTED\n a2.save(update_fields=[\"transcription_status\"])\n\n response = self.client.get(\n reverse(\"topic-detail\", args=(topic.slug,)),\n {\"transcription_status\": \"submitted\"},\n )\n self.assertEqual(response.status_code, 200)\n self.assertIn(\"sublevel_querystring\", response.context)\n self.assertEqual(\n response.context[\"sublevel_querystring\"], \"transcription_status=submitted\"\n )\n\n # Project WITH a matching url_filter should be included (2 links: image + title)\n self.assertContains(\n response,\n f\"/campaigns/{campaign.slug}/{project_with_filter.slug}/?transcription_status=submitted\",\n 2,\n )\n\n # Project WITHOUT a url_filter should also be included (also 2 links)\n self.assertContains(\n response,\n f\"/campaigns/{campaign.slug}/{project_without_filter.slug}/?transcription_status=submitted\",\n 2,\n )\n\n def test_topic_detail_with_invalid_status_ignores_filter(self):\n \"\"\"\n If transcription_status is present but invalid, the view should treat it\n as absent: no filtering by status and no sublevel_querystring.\n \"\"\"\n topic = create_topic(title=\"Filter Topic\", slug=\"filter-topic\")\n campaign = create_campaign(title=\"Filter Test Campaign\", slug=\"filter-test\")\n\n project_with_filter = create_project(\n campaign=campaign, title=\"Project With Filter\", slug=\"with-filter\"\n )\n project_without_filter = create_project(\n campaign=campaign, title=\"Project Without Filter\", slug=\"without-filter\"\n )\n\n ProjectTopic.objects.create(\n project=project_with_filter,\n topic=topic,\n url_filter=TranscriptionStatus.SUBMITTED,\n )\n ProjectTopic.objects.create(\n project=project_without_filter,\n topic=topic,\n url_filter=None,\n )\n\n # Make both projects eligible to display\n create_asset(item=create_item(project=project_with_filter))\n create_asset(item=create_item(project=project_without_filter))\n\n # Supply an invalid status\n response = self.client.get(\n reverse(\"topic-detail\", args=(topic.slug,)),\n {\"transcription_status\": \"not-a-real-status\", \"another_param\": \"x\"},\n )\n self.assertEqual(response.status_code, 200)\n\n # sublevel_querystring should be empty (invalid status ignored)\n self.assertIn(\"sublevel_querystring\", response.context)\n self.assertEqual(response.context[\"sublevel_querystring\"], \"\")\n\n # Project WITH a url_filter should use its own filter in links\n self.assertContains(\n response,\n f\"/campaigns/{campaign.slug}/{project_with_filter.slug}/?transcription_status=submitted\",\n 2,\n )\n # Project WITHOUT a url_filter should not include a transcription_status param\n self.assertNotContains(\n response,\n f\"/campaigns/{campaign.slug}/{project_without_filter.slug}/?transcription_status=\",\n )\n\n def test_url_filter_empty_string_treated_as_missing(self):\n topic = create_topic(title=\"Filter Topic\", slug=\"filter-topic\")\n campaign = create_campaign(title=\"Filter Test Campaign\", slug=\"filter-test\")\n\n project_empty_filter = create_project(\n campaign=campaign, title=\"Project Empty Filter\", slug=\"empty-filter\"\n )\n project_none_filter = create_project(\n campaign=campaign, title=\"Project None Filter\", slug=\"none-filter\"\n )\n\n ProjectTopic.objects.create(\n project=project_empty_filter, topic=topic, url_filter=\"\"\n )\n ProjectTopic.objects.create(\n project=project_none_filter, topic=topic, url_filter=None\n )\n\n # Make both eligible\n item_empty = create_item(project=project_empty_filter)\n asset_empty = create_asset(item=item_empty)\n item_none = create_item(project=project_none_filter)\n asset_none = create_asset(item=item_none)\n\n # no sublevel filter, so neither link should have transcription_status\n resp1 = self.client.get(reverse(\"topic-detail\", args=(topic.slug,)))\n self.assertEqual(resp1.status_code, 200)\n self.assertNotContains(\n resp1,\n f\"/campaigns/{campaign.slug}/{project_empty_filter.slug}/?transcription_status=\",\n )\n self.assertNotContains(\n resp1,\n f\"/campaigns/{campaign.slug}/{project_none_filter.slug}/?transcription_status=\",\n )\n\n # Set at least one asset to SUBMITTED for each project (so theyβre not excluded)\n asset_empty.transcription_status = TranscriptionStatus.SUBMITTED\n asset_empty.save(update_fields=[\"transcription_status\"])\n\n asset_none.transcription_status = TranscriptionStatus.SUBMITTED\n asset_none.save(update_fields=[\"transcription_status\"])\n\n # valid sublevel filter, so both included and use that status in links\n resp2 = self.client.get(\n reverse(\"topic-detail\", args=(topic.slug,)),\n {\"transcription_status\": \"submitted\"},\n )\n self.assertEqual(resp2.status_code, 200)\n self.assertContains(\n resp2,\n f\"/campaigns/{campaign.slug}/{project_empty_filter.slug}/?transcription_status=submitted\",\n 2,\n )\n self.assertContains(\n resp2,\n f\"/campaigns/{campaign.slug}/{project_none_filter.slug}/?transcription_status=submitted\",\n 2,\n )\n"
},
{
"path": "concordia/tests/test_views_transcription_review.py",
"content": "from django.core.cache import caches\nfrom django.db.models.signals import post_save\nfrom django.test import (\n TransactionTestCase,\n override_settings,\n)\nfrom django.urls import reverse\nfrom django.utils.timezone import now\n\nfrom concordia.models import (\n Asset,\n Transcription,\n TranscriptionStatus,\n)\nfrom concordia.signals.handlers import on_transcription_save\nfrom concordia.utils import get_anonymous_user\nfrom configuration.models import Configuration\n\nfrom .utils import (\n CreateTestUsers,\n JSONAssertMixin,\n create_asset,\n create_transcription,\n)\n\n\n@override_settings(\n RATELIMIT_ENABLE=False, SESSION_ENGINE=\"django.contrib.sessions.backends.cache\"\n)\nclass ReviewTranscriptionViewTests(\n CreateTestUsers, JSONAssertMixin, TransactionTestCase\n):\n def test_transcription_review(self):\n asset = create_asset()\n\n anon = get_anonymous_user()\n\n t1 = Transcription(asset=asset, user=anon, text=\"test\", submitted=now())\n t1.full_clean()\n t1.save()\n\n self.login_user()\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t1.pk,)), data={\"action\": \"foobar\"}\n )\n data = self.assertValidJSON(resp, expected_status=400)\n self.assertIn(\"error\", data)\n\n self.assertEqual(\n 1, Transcription.objects.filter(pk=t1.pk, accepted__isnull=True).count()\n )\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t1.pk,)), data={\"action\": \"accept\"}\n )\n data = self.assertValidJSON(resp, expected_status=200)\n\n self.assertEqual(\n 1, Transcription.objects.filter(pk=t1.pk, accepted__isnull=False).count()\n )\n\n def test_transcription_review_rate_limit(self):\n for cache in caches.all():\n cache.clear()\n anon = get_anonymous_user()\n self.login_user()\n try:\n config = Configuration.objects.get(key=\"review_rate_limit\")\n config.value = \"4\"\n config.data_type = Configuration.DataType.NUMBER\n config.save()\n except Configuration.DoesNotExist:\n Configuration.objects.create(\n key=\"review_rate_limit\",\n value=\"4\",\n data_type=Configuration.DataType.NUMBER,\n )\n\n Configuration.objects.get_or_create(\n key=\"review_rate_limit_popup_message\",\n defaults={\n \"value\": \"Test message\",\n \"data_type\": Configuration.DataType.HTML,\n },\n )\n Configuration.objects.get_or_create(\n key=\"review_rate_limit_popup_title\",\n defaults={\n \"value\": \"Test message\",\n \"data_type\": Configuration.DataType.HTML,\n },\n )\n Configuration.objects.get_or_create(\n key=\"review_rate_limit_banner_message\",\n defaults={\n \"value\": \"Test message\",\n \"data_type\": Configuration.DataType.HTML,\n },\n )\n\n asset = create_asset()\n t1 = create_transcription(user=anon, asset=asset)\n t2 = create_transcription(\n user=anon, asset=create_asset(item=asset.item, slug=\"test-asset-2\")\n )\n t3 = create_transcription(\n user=anon, asset=create_asset(item=asset.item, slug=\"test-asset-3\")\n )\n t4 = create_transcription(\n user=anon, asset=create_asset(item=asset.item, slug=\"test-asset-4\")\n )\n t5 = create_transcription(\n user=anon, asset=create_asset(item=asset.item, slug=\"test-asset-5\")\n )\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t1.pk,)), data={\"action\": \"accept\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t2.pk,)), data={\"action\": \"accept\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t3.pk,)), data={\"action\": \"accept\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t4.pk,)), data={\"action\": \"accept\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t5.pk,)), data={\"action\": \"accept\"}\n )\n data = self.assertValidJSON(resp, expected_status=429)\n self.assertIn(\"error\", data)\n\n def test_transcription_review_rate_limit_superuser(self):\n for cache in caches.all():\n cache.clear()\n anon = get_anonymous_user()\n self.user = self.create_super_user()\n self.login_user()\n try:\n config = Configuration.objects.get(key=\"review_rate_limit\")\n config.value = \"4\"\n config.data_type = Configuration.DataType.NUMBER\n config.save()\n except Configuration.DoesNotExist:\n Configuration.objects.create(\n key=\"review_rate_limit\",\n value=\"4\",\n data_type=Configuration.DataType.NUMBER,\n )\n\n Configuration.objects.get_or_create(\n key=\"review_rate_limit_popup_message\",\n defaults={\n \"value\": \"Test message\",\n \"data_type\": Configuration.DataType.HTML,\n },\n )\n Configuration.objects.get_or_create(\n key=\"review_rate_limit_popup_title\",\n defaults={\n \"value\": \"Test message\",\n \"data_type\": Configuration.DataType.HTML,\n },\n )\n Configuration.objects.get_or_create(\n key=\"review_rate_limit_banner_message\",\n defaults={\n \"value\": \"Test message\",\n \"data_type\": Configuration.DataType.HTML,\n },\n )\n\n asset = create_asset()\n t1 = create_transcription(user=anon, asset=asset)\n t2 = create_transcription(\n user=anon, asset=create_asset(item=asset.item, slug=\"test-asset-2\")\n )\n t3 = create_transcription(\n user=anon, asset=create_asset(item=asset.item, slug=\"test-asset-3\")\n )\n t4 = create_transcription(\n user=anon, asset=create_asset(item=asset.item, slug=\"test-asset-4\")\n )\n t5 = create_transcription(\n user=anon, asset=create_asset(item=asset.item, slug=\"test-asset-5\")\n )\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t1.pk,)), data={\"action\": \"accept\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t2.pk,)), data={\"action\": \"accept\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t3.pk,)), data={\"action\": \"accept\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t4.pk,)), data={\"action\": \"accept\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t5.pk,)), data={\"action\": \"accept\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n def test_transcription_review_asset_status_updates(self):\n \"\"\"\n Confirm that the Asset.transcription_status field is correctly updated\n throughout the review process\n \"\"\"\n asset = create_asset()\n\n anon = get_anonymous_user()\n\n # We should see NOT_STARTED only when no transcription records exist:\n self.assertEqual(asset.transcription_set.count(), 0)\n self.assertEqual(\n Asset.objects.get(pk=asset.pk).transcription_status,\n TranscriptionStatus.NOT_STARTED,\n )\n\n t1 = Transcription(asset=asset, user=anon, text=\"test\", submitted=now())\n t1.full_clean()\n t1.save()\n\n self.assertEqual(\n Asset.objects.get(pk=asset.pk).transcription_status,\n TranscriptionStatus.SUBMITTED,\n )\n\n # βLoginβ so we can review the anonymous transcription:\n self.login_user()\n\n self.assertEqual(\n 1, Transcription.objects.filter(pk=t1.pk, accepted__isnull=True).count()\n )\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t1.pk,)), data={\"action\": \"reject\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n # After rejecting a transcription, the asset status should be reset to\n # in-progress:\n self.assertEqual(\n 1,\n Transcription.objects.filter(\n pk=t1.pk, accepted__isnull=True, rejected__isnull=False\n ).count(),\n )\n self.assertEqual(\n Asset.objects.get(pk=asset.pk).transcription_status,\n TranscriptionStatus.IN_PROGRESS,\n )\n\n # We'll simulate a second attempt:\n\n t2 = Transcription(\n asset=asset, user=anon, text=\"test\", submitted=now(), supersedes=t1\n )\n t2.full_clean()\n t2.save()\n\n self.assertEqual(\n Asset.objects.get(pk=asset.pk).transcription_status,\n TranscriptionStatus.SUBMITTED,\n )\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t2.pk,)), data={\"action\": \"accept\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n\n self.assertEqual(\n 1, Transcription.objects.filter(pk=t2.pk, accepted__isnull=False).count()\n )\n self.assertEqual(\n Asset.objects.get(pk=asset.pk).transcription_status,\n TranscriptionStatus.COMPLETED,\n )\n\n def test_transcription_disallow_self_review(self):\n asset = create_asset()\n\n self.login_user()\n\n t1 = Transcription(asset=asset, user=self.user, text=\"test\", submitted=now())\n t1.full_clean()\n t1.save()\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t1.pk,)), data={\"action\": \"accept\"}\n )\n data = self.assertValidJSON(resp, expected_status=400)\n self.assertIn(\"error\", data)\n self.assertEqual(\"You cannot accept your own transcription\", data[\"error\"])\n\n def test_transcription_allow_self_reject(self):\n asset = create_asset()\n\n self.login_user()\n\n t1 = Transcription(asset=asset, user=self.user, text=\"test\", submitted=now())\n t1.full_clean()\n t1.save()\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t1.pk,)), data={\"action\": \"reject\"}\n )\n self.assertValidJSON(resp, expected_status=200)\n self.assertEqual(\n Asset.objects.get(pk=asset.pk).transcription_status,\n TranscriptionStatus.IN_PROGRESS,\n )\n self.assertEqual(Transcription.objects.get(pk=t1.pk).reviewed_by, self.user)\n\n def test_transcription_double_review(self):\n asset = create_asset()\n\n anon = get_anonymous_user()\n\n t1 = Transcription(asset=asset, user=anon, text=\"test\", submitted=now())\n t1.full_clean()\n t1.save()\n\n self.login_user()\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t1.pk,)), data={\"action\": \"accept\"}\n )\n data = self.assertValidJSON(resp, expected_status=200)\n\n resp = self.client.post(\n reverse(\"review-transcription\", args=(t1.pk,)), data={\"action\": \"reject\"}\n )\n data = self.assertValidJSON(resp, expected_status=400)\n self.assertIn(\"error\", data)\n self.assertEqual(\"This transcription has already been reviewed\", data[\"error\"])\n\n def tearDown(self):\n # We'll test the signal handler separately\n post_save.connect(on_transcription_save, sender=Transcription)\n"
},
{
"path": "concordia/tests/test_views_transcription_save.py",
"content": "import sys\nfrom unittest.mock import patch\n\nfrom django import forms\nfrom django.db.models.signals import post_save\nfrom django.test import (\n TransactionTestCase,\n override_settings,\n)\nfrom django.urls import reverse\n\nfrom concordia.models import (\n Transcription,\n)\nfrom concordia.signals.handlers import on_transcription_save\n\nfrom .utils import (\n CreateTestUsers,\n JSONAssertMixin,\n create_asset,\n)\n\n\n@override_settings(\n RATELIMIT_ENABLE=False, SESSION_ENGINE=\"django.contrib.sessions.backends.cache\"\n)\nclass SaveTranscriptionViewTests(CreateTestUsers, JSONAssertMixin, TransactionTestCase):\n def setUp(self):\n self.asset = create_asset()\n\n def test_turnstile_validation_fails(self):\n # Test when Turnstile validation failes\n with patch(\"concordia.turnstile.fields.TurnstileField.validate\") as mock:\n mock.side_effect = forms.ValidationError(\n \"Testing error\", code=\"invalid_turnstile\"\n )\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"test\"},\n )\n data = self.assertValidJSON(resp, expected_status=401)\n self.assertIn(\"error\", data)\n\n def test_initial_save_success(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"test\"},\n )\n data = self.assertValidJSON(resp, expected_status=201)\n self.assertIn(\"submissionUrl\", data)\n\n def test_duplicate_without_supersedes_conflict(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"test\"},\n )\n # Test attempts to create a second transcription without marking that it\n # supersedes the previous one:\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"test\"},\n )\n data = self.assertValidJSON(resp, expected_status=409)\n self.assertIn(\"error\", data)\n\n def test_save_with_url_error(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"test\"},\n )\n # If a transcription contains a URL, it should return an error\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\n \"text\": \"http://example.com\",\n \"supersedes\": self.asset.transcription_set.get().pk,\n },\n )\n data = self.assertValidJSON(resp, expected_status=400)\n self.assertIn(\"error\", data)\n\n def test_unacceptable_characters_are_removed_on_save(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n bad_text = \"He\\u200bllo\\tWorld\\xa0\\u3000\\u2003!\\nBad\\x00Char\\x1fHere\\u200b\"\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": bad_text},\n )\n data = self.assertValidJSON(resp, expected_status=201)\n self.assertIn(\"submissionUrl\", data)\n t = self.asset.transcription_set.get()\n self.assertEqual(t.text, \"Hello\\tWorld\\xa0\\u3000\\u2003!\\nBadCharHere\")\n\n def test_unacceptable_characters_are_removed_when_superseding(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"first\"},\n )\n bad_text = \"b\\u200bad\\x00\"\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\n \"text\": bad_text,\n \"supersedes\": self.asset.transcription_set.get().pk,\n },\n )\n data = self.assertValidJSON(resp, expected_status=201)\n self.assertIn(\"submissionUrl\", data)\n new_t = self.asset.transcription_set.order_by(\"pk\").last()\n self.assertEqual(new_t.text, \"bad\")\n\n def test_save_with_supersedes_success(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"test\"},\n )\n # Test that it correctly works when supersedes is set\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\n \"text\": \"test\",\n \"supersedes\": self.asset.transcription_set.get().pk,\n },\n )\n data = self.assertValidJSON(resp, expected_status=201)\n self.assertIn(\"submissionUrl\", data)\n\n def test_supersedes_sets_ocr_originated_when_previous_was_ocr_originated(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"test\"},\n )\n # Test that it correctly works when supersedes is set and confirm\n # ocr_originaed is properly set\n transcription = self.asset.transcription_set.order_by(\"pk\").last()\n transcription.ocr_originated = True\n transcription.save()\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\n \"text\": \"test\",\n \"supersedes\": self.asset.transcription_set.order_by(\"pk\").last().pk,\n },\n )\n data = self.assertValidJSON(resp, expected_status=201)\n self.assertIn(\"submissionUrl\", data)\n new_transcription = self.asset.transcription_set.order_by(\"pk\").last()\n self.assertTrue(new_transcription.ocr_originated)\n\n def test_supersede_already_superseded_conflict(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n first_resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"test\"},\n )\n self.assertValidJSON(first_resp, expected_status=201)\n first_pk = self.asset.transcription_set.order_by(\"pk\").first().pk\n\n self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"test 2\", \"supersedes\": first_pk},\n )\n\n # We should see an error if you attempt to supersede a transcription\n # which has already been superseded:\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\n \"text\": \"test\",\n \"supersedes\": self.asset.transcription_set.order_by(\"pk\")\n .first()\n .pk,\n },\n )\n data = self.assertValidJSON(resp, expected_status=409)\n self.assertIn(\"error\", data)\n\n def test_supersede_nonexistent_returns_400(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n # We should get an error if you attempt to supersede a transcription\n # that doesn't exist\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\n \"text\": \"test\",\n \"supersedes\": sys.maxsize,\n },\n )\n data = self.assertValidJSON(resp, expected_status=400)\n self.assertIn(\"error\", data)\n\n def test_supersede_invalid_pk_returns_400(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n # We should get an error if you attempt to supersede with\n # with a pk that is invalid (i.e., a string instead of int)\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\n \"text\": \"test\",\n \"supersedes\": \"bad-pk\",\n },\n )\n data = self.assertValidJSON(resp, expected_status=400)\n self.assertIn(\"error\", data)\n\n def test_logged_in_user_can_take_over_from_anonymous(self):\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n anon_resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\"text\": \"test\"},\n )\n self.assertValidJSON(anon_resp, expected_status=201)\n\n # A logged in user can take over from an anonymous user:\n self.login_user()\n resp = self.client.post(\n reverse(\"save-transcription\", args=(self.asset.pk,)),\n data={\n \"text\": \"test\",\n \"supersedes\": self.asset.transcription_set.order_by(\"pk\").last().pk,\n },\n )\n data = self.assertValidJSON(resp, expected_status=201)\n self.assertIn(\"submissionUrl\", data)\n\n def tearDown(self):\n # We'll test the signal handler separately\n post_save.connect(on_transcription_save, sender=Transcription)\n"
},
{
"path": "concordia/tests/test_views_transcription_submit.py",
"content": "from unittest.mock import patch\n\nfrom django import forms\nfrom django.db.models.signals import post_save\nfrom django.test import (\n TransactionTestCase,\n override_settings,\n)\nfrom django.urls import reverse\n\nfrom concordia.models import (\n Transcription,\n)\nfrom concordia.signals.handlers import on_transcription_save\nfrom concordia.utils import get_anonymous_user\n\nfrom .utils import (\n CreateTestUsers,\n JSONAssertMixin,\n create_asset,\n)\n\n\n@override_settings(\n RATELIMIT_ENABLE=False, SESSION_ENGINE=\"django.contrib.sessions.backends.cache\"\n)\nclass SubmitTranscriptionViewTests(\n CreateTestUsers, JSONAssertMixin, TransactionTestCase\n):\n def test_anonymous_transcription_submission(self):\n asset = create_asset()\n anon = get_anonymous_user()\n\n transcription = Transcription(asset=asset, user=anon, text=\"previous entry\")\n transcription.full_clean()\n transcription.save()\n\n with patch(\"concordia.turnstile.fields.TurnstileField.validate\") as mock:\n mock.side_effect = forms.ValidationError(\n \"Testing error\", code=\"invalid_turnstile\"\n )\n resp = self.client.post(\n reverse(\"submit-transcription\", args=(transcription.pk,))\n )\n data = self.assertValidJSON(resp, expected_status=401)\n self.assertIn(\"error\", data)\n\n self.assertFalse(Transcription.objects.filter(submitted__isnull=False).exists())\n\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n self.client.post(\n reverse(\"submit-transcription\", args=(transcription.pk,)),\n )\n self.assertTrue(\n Transcription.objects.filter(submitted__isnull=False).exists()\n )\n\n def test_transcription_submission(self):\n asset = create_asset()\n\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n resp = self.client.post(\n reverse(\"save-transcription\", args=(asset.pk,)), data={\"text\": \"test\"}\n )\n data = self.assertValidJSON(resp, expected_status=201)\n\n transcription = Transcription.objects.get()\n self.assertIsNone(transcription.submitted)\n\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n resp = self.client.post(\n reverse(\"submit-transcription\", args=(transcription.pk,))\n )\n data = self.assertValidJSON(resp, expected_status=200)\n self.assertIn(\"id\", data)\n self.assertEqual(data[\"id\"], transcription.pk)\n\n transcription = Transcription.objects.get()\n self.assertTrue(transcription.submitted)\n\n def test_stale_transcription_submission(self):\n asset = create_asset()\n\n anon = get_anonymous_user()\n\n t1 = Transcription(asset=asset, user=anon, text=\"test\")\n t1.full_clean()\n t1.save()\n\n t2 = Transcription(asset=asset, user=anon, text=\"test\", supersedes=t1)\n t2.full_clean()\n t2.save()\n\n with patch(\n \"concordia.turnstile.fields.TurnstileField.validate\", return_value=True\n ):\n resp = self.client.post(reverse(\"submit-transcription\", args=(t1.pk,)))\n data = self.assertValidJSON(resp, expected_status=400)\n self.assertIn(\"error\", data)\n\n def tearDown(self):\n # We'll test the signal handler separately\n post_save.connect(on_transcription_save, sender=Transcription)\n"
},
{
"path": "concordia/tests/test_views_utils.py",
"content": "import datetime\nfrom time import time\n\nfrom django.contrib.auth.models import AnonymousUser\nfrom django.test import RequestFactory, TestCase, override_settings\nfrom django.utils.timezone import make_aware, now\n\nfrom concordia.models import (\n Asset,\n Transcription,\n TranscriptionStatus,\n)\nfrom concordia.views.utils import (\n AnonymousUserValidationCheckMixin,\n _get_pages,\n annotate_children_with_progress_stats,\n calculate_asset_stats,\n)\n\nfrom .utils import (\n CreateTestUsers,\n create_asset,\n create_campaign,\n create_item,\n create_project,\n create_transcription,\n)\n\n\nclass GetPagesTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.factory = RequestFactory()\n self.user = self.create_test_user()\n\n # Base campaign, project, item setup\n self.campaign = create_campaign(slug=\"gp-camp\", title=\"gp-camp\")\n self.project = create_project(\n campaign=self.campaign, slug=\"gp-proj\", title=\"gp-proj\"\n )\n self.item = create_item(project=self.project, item_id=\"gp-item\")\n\n # Two assets in the same item\n self.asset1 = create_asset(item=self.item, slug=\"gp-a1\", sequence=1)\n self.asset2 = create_asset(item=self.item, slug=\"gp-a2\", sequence=2)\n\n # Another campaign and project for campaign filtering tests\n self.campaign2 = create_campaign(slug=\"gp-camp-2\", title=\"gp-camp-2\")\n self.project2 = create_project(\n campaign=self.campaign2, slug=\"gp-proj-2\", title=\"gp-proj-2\"\n )\n self.item2 = create_item(project=self.project2, item_id=\"gp-item-2\")\n self.asset3_other_campaign = create_asset(\n item=self.item2, slug=\"gp-a3\", sequence=1\n )\n\n def _request(self, params: dict[str, str]):\n request = self.factory.get(\"/dummy\", data=params)\n request.user = self.user\n return request\n\n def _touch_transcription_times(\n self,\n transcription: Transcription,\n *,\n created_on=None,\n updated_on=None,\n reviewer=None,\n ):\n if reviewer is not None:\n transcription.reviewed_by = reviewer\n if created_on is not None:\n transcription.created_on = created_on\n if updated_on is not None:\n transcription.updated_on = updated_on\n transcription.save(update_fields=[\"reviewed_by\", \"created_on\", \"updated_on\"])\n\n def test_activity_filters_transcribed_vs_reviewed_vs_default(self):\n now_reference = now()\n\n # asset1: transcribed by self.user\n transcription1 = create_transcription(asset=self.asset1, user=self.user)\n self._touch_transcription_times(\n transcription1, created_on=now_reference - datetime.timedelta(hours=2)\n )\n\n # asset2: reviewed by self.user\n transcription2 = create_transcription(\n asset=self.asset2, user=self.create_test_user(\"transcriber\")\n )\n self._touch_transcription_times(\n transcription2,\n reviewer=self.user,\n updated_on=now_reference - datetime.timedelta(hours=1),\n )\n\n # Default behavior includes both\n queryset_default = _get_pages(self._request({}))\n self.assertCountEqual(\n list(queryset_default.values_list(\"id\", flat=True)),\n [self.asset1.id, self.asset2.id],\n )\n\n # Transcribed only\n queryset_transcribed = _get_pages(self._request({\"activity\": \"transcribed\"}))\n self.assertListEqual(\n list(queryset_transcribed.values_list(\"id\", flat=True)),\n [self.asset1.id],\n )\n\n # Reviewed only\n queryset_reviewed = _get_pages(self._request({\"activity\": \"reviewed\"}))\n self.assertListEqual(\n list(queryset_reviewed.values_list(\"id\", flat=True)),\n [self.asset2.id],\n )\n\n def test_status_filter_exclusions(self):\n # Ensure the user is associated with each asset via transcriptions\n create_transcription(asset=self.asset1, user=self.user)\n create_transcription(asset=self.asset2, user=self.user, submitted=now())\n\n # Mark asset1 as IN_PROGRESS explicitly\n Asset.objects.filter(pk=self.asset1.pk).update(\n transcription_status=TranscriptionStatus.IN_PROGRESS\n )\n\n # Also add an asset in COMPLETED with a user transcription\n completed_asset = create_asset(item=self.item, slug=\"gp-a4\", sequence=3)\n create_transcription(asset=completed_asset, user=self.user)\n Asset.objects.filter(pk=completed_asset.pk).update(\n transcription_status=TranscriptionStatus.COMPLETED\n )\n\n # Only \"submitted\" requested, so exclude IN_PROGRESS and COMPLETED\n queryset = _get_pages(self._request({\"status\": \"submitted\"}))\n self.assertListEqual(\n list(queryset.values_list(\"id\", flat=True)), [self.asset2.id]\n )\n\n def test_date_range_and_single_day_filters_and_ordering(self):\n # Transcriptions (associate user) with distinct activity dates\n today = now()\n day_minus_3 = make_aware(\n datetime.datetime.combine(\n (today - datetime.timedelta(days=3)).date(), datetime.time(12)\n )\n )\n day_minus_1 = make_aware(\n datetime.datetime.combine(\n (today - datetime.timedelta(days=1)).date(), datetime.time(12)\n )\n )\n\n transcription1 = create_transcription(asset=self.asset1, user=self.user)\n self._touch_transcription_times(\n transcription1, created_on=day_minus_3, updated_on=day_minus_3\n )\n\n transcription2 = create_transcription(asset=self.asset2, user=self.user)\n self._touch_transcription_times(\n transcription2, created_on=day_minus_1, updated_on=day_minus_1\n )\n\n # The range filter from two days ago through today should include\n # asset2 (day minus one) and exclude asset1 (day minus three)\n start = (today - datetime.timedelta(days=2)).strftime(\"%Y-%m-%d\")\n end = today.strftime(\"%Y-%m-%d\")\n queryset_range = _get_pages(self._request({\"start\": start, \"end\": end}))\n self.assertListEqual(\n list(queryset_range.values_list(\"id\", flat=True)), [self.asset2.id]\n )\n\n # A single-day filter for day minus three picks asset1\n only_day = (today - datetime.timedelta(days=3)).strftime(\"%Y-%m-%d\")\n queryset_single = _get_pages(self._request({\"start\": only_day}))\n self.assertListEqual(\n list(queryset_single.values_list(\"id\", flat=True)), [self.asset1.id]\n )\n\n # Ordering: ascending vs default (descending)\n queryset_ascending = _get_pages(self._request({\"order_by\": \"date-ascending\"}))\n self.assertEqual(\n list(queryset_ascending.values_list(\"id\", flat=True)),\n [self.asset1.id, self.asset2.id],\n )\n\n queryset_descending = _get_pages(self._request({}))\n self.assertEqual(\n list(queryset_descending.values_list(\"id\", flat=True)),\n [self.asset2.id, self.asset1.id],\n )\n\n def test_campaign_filter_and_six_month_cutoff(self):\n # Link user to assets in both campaigns\n recent_timestamp = now() - datetime.timedelta(days=5)\n old_timestamp = now() - datetime.timedelta(days=6 * 30 + 10)\n\n # Asset in base campaign (recent)\n transcription1 = create_transcription(asset=self.asset1, user=self.user)\n self._touch_transcription_times(\n transcription1, created_on=recent_timestamp, updated_on=recent_timestamp\n )\n\n # Asset in other campaign (recent)\n transcription2 = create_transcription(\n asset=self.asset3_other_campaign, user=self.user\n )\n self._touch_transcription_times(\n transcription2, created_on=recent_timestamp, updated_on=recent_timestamp\n )\n\n # Very old activity on asset2 so it should be filtered out by the\n # six months cutoff\n transcription_old = create_transcription(asset=self.asset2, user=self.user)\n self._touch_transcription_times(\n transcription_old, created_on=old_timestamp, updated_on=old_timestamp\n )\n\n # Without a campaign filter, both recent assets are present\n # and the old one is excluded\n queryset = _get_pages(self._request({}))\n asset_ids = set(queryset.values_list(\"id\", flat=True))\n self.assertSetEqual(asset_ids, {self.asset1.id, self.asset3_other_campaign.id})\n\n # The campaign filter picks only the other campaign's asset\n queryset_campaign2 = _get_pages(\n self._request({\"campaign\": str(self.campaign2.pk)})\n )\n self.assertListEqual(\n list(queryset_campaign2.values_list(\"id\", flat=True)),\n [self.asset3_other_campaign.id],\n )\n\n def test_status_filter_includes_completed_when_requested(self):\n \"\"\"\n When \"completed\" is requested, completed assets are kept while\n submitted and in progress assets are excluded.\n \"\"\"\n # Prepare three assets that all have activity from this user.\n completed_asset = create_asset(\n item=self.item, slug=\"gp-a4-completed\", sequence=4\n )\n create_transcription(asset=completed_asset, user=self.user)\n Asset.objects.filter(pk=completed_asset.pk).update(\n transcription_status=TranscriptionStatus.COMPLETED\n )\n\n create_transcription(asset=self.asset1, user=self.user)\n Asset.objects.filter(pk=self.asset1.pk).update(\n transcription_status=TranscriptionStatus.IN_PROGRESS\n )\n\n create_transcription(\n asset=self.asset2, user=self.user, submitted=now()\n ) # submitted\n\n # Request only \"completed\" status.\n queryset = _get_pages(self._request({\"status\": \"completed\"}))\n self.assertListEqual(\n list(queryset.values_list(\"id\", flat=True)), [completed_asset.id]\n )\n\n def test_status_filter_includes_in_progress_and_excludes_submitted_not_requested(\n self,\n ):\n \"\"\"\n When \"in_progress\" is requested, in progress assets are kept and\n submitted assets are excluded because \"submitted\" is not requested.\n \"\"\"\n # Prepare one in progress and one submitted asset with this user's activity.\n create_transcription(asset=self.asset1, user=self.user)\n Asset.objects.filter(pk=self.asset1.pk).update(\n transcription_status=TranscriptionStatus.IN_PROGRESS\n )\n\n create_transcription(\n asset=self.asset2, user=self.user, submitted=now()\n ) # submitted\n\n # Request only \"in_progress\" status.\n queryset = _get_pages(self._request({\"status\": \"in_progress\"}))\n ids = list(queryset.values_list(\"id\", flat=True))\n self.assertIn(self.asset1.id, ids)\n self.assertNotIn(self.asset2.id, ids)\n\n\nclass CalculateAssetStatsTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.user = self.create_test_user()\n self.campaign = create_campaign(slug=\"cas-c\", title=\"cas-c\")\n self.project = create_project(\n campaign=self.campaign, slug=\"cas-p\", title=\"cas-p\"\n )\n self.item = create_item(project=self.project, item_id=\"cas-i\")\n\n def test_counts_percents_and_contributors_remove_none_branch(self):\n # Build a small asset set with varied statuses.\n asset_not_started = create_asset(item=self.item, slug=\"cas-ns\", sequence=1)\n asset_in_progress = create_asset(item=self.item, slug=\"cas-ip\", sequence=2)\n asset_submitted = create_asset(item=self.item, slug=\"cas-sub\", sequence=3)\n\n # Set desired statuses directly.\n Asset.objects.filter(pk=asset_not_started.pk).update(\n transcription_status=TranscriptionStatus.NOT_STARTED\n )\n Asset.objects.filter(pk=asset_in_progress.pk).update(\n transcription_status=TranscriptionStatus.IN_PROGRESS\n )\n Asset.objects.filter(pk=asset_submitted.pk).update(\n transcription_status=TranscriptionStatus.SUBMITTED\n )\n\n # Create transcriptions ONLY for the assets that should not remain NOT_STARTED.\n # For IN_PROGRESS, a plain transcription moves or keeps the asset in progress.\n transcription_in_progress = create_transcription(\n asset=asset_in_progress, user=self.user\n )\n # For SUBMITTED, mark the transcription as submitted so the\n # signal preserves SUBMITTED.\n transcription_submitted = create_transcription(\n asset=asset_submitted, user=self.user, submitted=now()\n )\n # Ensure there is at least one None in reviewed_by so the remove(None)\n # path is exercised.\n Transcription.objects.filter(\n pk__in=[transcription_in_progress.pk, transcription_submitted.pk]\n ).update(reviewed_by=None)\n\n context = {}\n calculate_asset_stats(\n Asset.objects.filter(\n pk__in=[asset_not_started.pk, asset_in_progress.pk, asset_submitted.pk]\n ),\n context,\n )\n\n # contributor_count counts unique user_ids and reviewed_by values, minus None.\n self.assertEqual(context[\"contributor_count\"], 1)\n\n # Counts per status.\n self.assertEqual(context[\"not_started_count\"], 1)\n self.assertEqual(context[\"in_progress_count\"], 1)\n self.assertEqual(context[\"submitted_count\"], 1)\n # COMPLETED not present.\n self.assertEqual(context.get(\"completed_count\", 0), 0)\n\n # Percentages should round sensibly for 1 out of 3.\n self.assertEqual(context[\"not_started_percent\"], round(100 * (1 / 3)))\n self.assertEqual(context[\"in_progress_percent\"], round(100 * (1 / 3)))\n self.assertEqual(context[\"submitted_percent\"], round(100 * (1 / 3)))\n\n # Labeled list populated and includes \"not_started\".\n self.assertTrue(\n any(\n status_key == \"not_started\"\n for status_key, _, _ in context[\"transcription_status_counts\"]\n )\n )\n\n def test_contributors_keyerror_branch_and_cap_99(self):\n # Create 100 assets and set 99 to NOT_STARTED and 1 to IN_PROGRESS.\n assets = []\n for i in range(1, 101):\n a = create_asset(item=self.item, slug=f\"cas-bulk-{i}\", sequence=i)\n assets.append(a)\n\n Asset.objects.filter(pk__in=[a.pk for a in assets[:99]]).update(\n transcription_status=TranscriptionStatus.NOT_STARTED\n )\n Asset.objects.filter(pk=assets[-1].pk).update(\n transcription_status=TranscriptionStatus.IN_PROGRESS\n )\n\n # Create a transcription ONLY for the single IN_PROGRESS asset\n # and set a reviewer. This ensures there is no None in reviewed_by,\n # which triggers the KeyError branch when calculate_asset_stats\n # attempts to remove(None) from the contributor set.\n other_user = self.create_test_user(username=\"cas-reviewer\")\n transcription = create_transcription(asset=assets[-1], user=self.user)\n transcription.reviewed_by = other_user\n transcription.save(update_fields=[\"reviewed_by\"])\n\n context = {}\n calculate_asset_stats(\n Asset.objects.filter(pk__in=[a.pk for a in assets]), context\n )\n\n # Two distinct contributors: the creator (self.user) and\n # the reviewer (other_user).\n self.assertEqual(context[\"contributor_count\"], 2)\n\n # Verify percentages and that the 99 percent capping behavior is applied.\n self.assertEqual(context[\"not_started_percent\"], 99)\n self.assertEqual(context[\"in_progress_percent\"], 1)\n self.assertEqual(context.get(\"submitted_percent\", 0), 0)\n self.assertEqual(context.get(\"completed_percent\", 0), 0)\n\n # Also verify counts to ensure the underlying distribution is as intended.\n self.assertEqual(context[\"not_started_count\"], 99)\n self.assertEqual(context[\"in_progress_count\"], 1)\n self.assertEqual(context.get(\"submitted_count\", 0), 0)\n self.assertEqual(context.get(\"completed_count\", 0), 0)\n\n\nclass AnnotateChildrenProgressStatsTests(TestCase):\n class Obj:\n pass\n\n def test_progress_stats_with_capping_and_lowest_status(self):\n obj = self.Obj()\n # Construct counts such that one bucket yields at least ninety nine\n # but less than one hundred percent\n obj.not_started_count = 99\n obj.in_progress_count = 1\n obj.submitted_count = 0\n obj.completed_count = 0\n\n annotate_children_with_progress_stats([obj])\n\n # Total\n self.assertEqual(obj.total_count, 100)\n # Capping at ninety nine\n self.assertEqual(obj.not_started_percent, 99)\n # Others\n self.assertEqual(obj.in_progress_percent, 1)\n self.assertEqual(obj.submitted_percent, 0)\n self.assertEqual(obj.completed_percent, 0)\n # Lowest is the first non-zero by CHOICES order; expect \"not_started\"\n self.assertEqual(obj.lowest_transcription_status, \"not_started\")\n\n def test_progress_stats_zero_total(self):\n obj = self.Obj()\n obj.not_started_count = 0\n obj.in_progress_count = 0\n obj.submitted_count = 0\n obj.completed_count = 0\n\n annotate_children_with_progress_stats([obj])\n\n self.assertEqual(obj.total_count, 0)\n self.assertEqual(obj.not_started_percent, 0)\n self.assertEqual(obj.in_progress_percent, 0)\n self.assertEqual(obj.submitted_percent, 0)\n self.assertEqual(obj.completed_percent, 0)\n self.assertIsNone(obj.lowest_transcription_status)\n\n\nclass _BaseView:\n \"\"\"\n Minimal base class that provides get_context_data so the mixin can call super().\n \"\"\"\n\n def get_context_data(self, **kwargs):\n return {}\n\n\nclass DummyTemplateView(AnonymousUserValidationCheckMixin, _BaseView):\n \"\"\"\n Stand-in view. The mixin is first in the MRO so its get_context_data runs,\n then it calls super() which resolves to _BaseView.get_context_data.\n \"\"\"\n\n pass\n\n\nclass AnonymousUserValidationCheckMixinTests(CreateTestUsers, TestCase):\n def setUp(self):\n self.factory = RequestFactory()\n self.user = self.create_test_user()\n\n def _attach_session(self, request):\n # Attach a session dictionary-like attribute without middleware dependency\n request.session = {}\n return request\n\n @override_settings(ANONYMOUS_USER_VALIDATION_INTERVAL=10)\n def test_unauthenticated_requires_validation_when_stale(self):\n request = self.factory.get(\"/dummy\")\n request.user = AnonymousUser()\n self._attach_session(request)\n # There is no prior validation so the default timestamp is zero\n # and the validation is stale\n view = DummyTemplateView()\n view.request = request\n context = view.get_context_data()\n self.assertTrue(context[\"anonymous_user_validation_required\"])\n\n @override_settings(ANONYMOUS_USER_VALIDATION_INTERVAL=10)\n def test_unauthenticated_recent_validation_is_not_required(self):\n request = self.factory.get(\"/dummy\")\n request.user = AnonymousUser()\n self._attach_session(request)\n request.session[\"turnstile_last_validated\"] = int(time())\n view = DummyTemplateView()\n view.request = request\n context = view.get_context_data()\n self.assertFalse(context[\"anonymous_user_validation_required\"])\n\n @override_settings(ANONYMOUS_USER_VALIDATION_INTERVAL=10)\n def test_authenticated_never_requires_validation(self):\n request = self.factory.get(\"/dummy\")\n request.user = self.user\n self._attach_session(request)\n view = DummyTemplateView()\n view.request = request\n context = view.get_context_data()\n self.assertFalse(context[\"anonymous_user_validation_required\"])\n"
},
{
"path": "concordia/tests/test_widgets.py",
"content": "from django.test import TestCase, override_settings\n\nfrom concordia.turnstile.widgets import TurnstileWidget\nfrom concordia.widgets import EmailWidget\n\n\nclass TestWidgets(TestCase):\n def test_EmailWidget(self):\n widget = EmailWidget()\n output = widget.render(\"email\", None)\n self.assertHTMLEqual(\n output, '/reviewable/\",\n views.campaigns.FilteredCampaignDetailView.as_view(),\n name=\"filtered-campaign-detail\",\n ),\n path(\n \"/\",\n views.campaigns.CampaignDetailView.as_view(),\n name=\"campaign-detail\",\n ),\n path(\n \"/export/csv/\",\n exporter_views.ExportCampaignToCSV.as_view(),\n name=\"campaign-export-csv\",\n ),\n path(\n \"/export/bagit/\",\n exporter_views.ExportCampaignToBagIt.as_view(),\n name=\"campaign-export-bagit\",\n ),\n path(\n \"//export/bagit/\",\n exporter_views.ExportProjectToBagIt.as_view(),\n name=\"project-export-bagit\",\n ),\n path(\n (\n \"//\"\n \"/export/bagit/\"\n ),\n exporter_views.ExportItemToBagIt.as_view(),\n name=\"item-export-bagit\",\n ),\n path(\n \"/report/\",\n views.campaigns.ReportCampaignView.as_view(),\n name=\"campaign-report\",\n ),\n path(\n \"///reviewable/\",\n views.items.FilteredItemDetailView.as_view(),\n name=\"filtered-item-detail\",\n ),\n path(\n (\n \"//\"\n \"//\"\n ),\n views.assets.AssetDetailView.as_view(),\n name=\"asset-detail\",\n ),\n # n.b. this must be above project-detail to avoid being seen as a project slug:\n path(\n \"/next-transcribable-asset/\",\n views.assets.redirect_to_next_transcribable_campaign_asset,\n name=\"redirect-to-next-transcribable-campaign-asset\",\n ),\n path(\n \"/next-reviewable-asset/\",\n views.assets.redirect_to_next_reviewable_campaign_asset,\n name=\"redirect-to-next-reviewable-campaign-asset\",\n ),\n path(\n \"//reviewable/\",\n views.projects.FilteredProjectDetailView.as_view(),\n name=\"filtered-project-detail\",\n ),\n path(\n \"//\",\n views.projects.ProjectDetailView.as_view(),\n name=\"project-detail\",\n ),\n path(\n \"///\",\n views.items.ItemDetailView.as_view(),\n name=\"item-detail\",\n ),\n ],\n \"transcriptions\",\n)\n\nurlpatterns = [\n path(\"\", views.HomeView.as_view(), name=\"homepage\"),\n path(\"healthz\", views.healthz, name=\"health-check\"),\n path(\"letter\", views.accounts.account_letter, name=\"user-letter\"),\n path(\"about/\", views.simple_pages.about_simple_page, name=\"about\"),\n # These patterns are to make sure various links to help-center URLs don't break\n # when the URLs are changed to not include help-center and can be removed after\n # all links are updated.\n path(\n \"help-center/\",\n RedirectView.as_view(pattern_name=\"welcome-guide\"),\n name=\"help-center\",\n ),\n path(\n \"help-center/welcome-guide/\", RedirectView.as_view(pattern_name=\"welcome-guide\")\n ),\n path(\n \"help-center/welcome-guide-esp/\",\n RedirectView.as_view(pattern_name=\"welcome-guide-spanish\"),\n ),\n path(\n \"help-center/-esp/\",\n views.simple_pages.HelpCenterSpanishRedirectView.as_view(),\n ),\n path(\n \"help-center//\",\n views.simple_pages.HelpCenterRedirectView.as_view(),\n ),\n # End of help-center patterns\n path(\"get-started/\", views.simple_pages.simple_page, name=\"welcome-guide\"),\n path(\n \"get-started/how-to-transcribe/\",\n views.simple_pages.simple_page,\n name=\"transcription-basic-rules\",\n ),\n path(\n \"get-started/how-to-review/\",\n views.simple_pages.simple_page,\n name=\"how-to-review\",\n ),\n path(\"get-started/how-to-tag/\", views.simple_pages.simple_page, name=\"how-to-tag\"),\n path(\n \"get-started//\", views.simple_pages.simple_page, name=\"simple-page\"\n ),\n path(\n \"get-started-esp/\",\n views.simple_pages.simple_page,\n name=\"welcome-guide-spanish\",\n ),\n path(\n \"get-started-esp/how-to-transcribe-esp/\",\n views.simple_pages.simple_page,\n name=\"how-to-transcribe-spanish\",\n ),\n path(\n \"get-started-esp/how-to-review-esp/\",\n views.simple_pages.simple_page,\n name=\"how-to-review-spanish\",\n ),\n path(\n \"get-started-esp/how-to-tag-esp/\",\n views.simple_pages.simple_page,\n name=\"how-to-tag-spanish\",\n ),\n path(\n \"get-started-esp//\",\n views.simple_pages.simple_page,\n name=\"simple-page-spanish\",\n ),\n path(\n \"for-educators/\",\n views.simple_pages.simple_page,\n name=\"for-educators\",\n ),\n path(\n \"for-staff/\",\n views.simple_pages.simple_page,\n name=\"for-staff\",\n ),\n path(\n \"resources/\",\n RedirectView.as_view(\n pattern_name=\"guidelines\", permanent=True, query_string=True\n ),\n name=\"resources\",\n ),\n path(\n \"service/\",\n views.simple_pages.simple_page,\n name=\"service\",\n ),\n path(\n \"guidelines/\",\n views.simple_pages.simple_page,\n name=\"guidelines\",\n ),\n path(\n \"programs/\",\n views.simple_pages.simple_page,\n name=\"programs\",\n ),\n path(\n \"latest/\",\n RedirectView.as_view(pattern_name=\"about\", permanent=True, query_string=True),\n ),\n path(\"questions/\", views.simple_pages.simple_page, name=\"questions\"),\n path(\n \"contact/\",\n RedirectView.as_view(url=\"https://ask.loc.gov/crowd\"),\n name=\"contact\",\n ),\n path(\n \"help-center/\",\n RedirectView.as_view(pattern_name=\"welcome-guide\"),\n name=\"help-center\",\n ),\n path(\n \"campaigns-topics/\",\n views.campaigns.CampaignTopicListView.as_view(),\n name=\"campaign-topic-list\",\n ),\n path(\n \"topics//\",\n views.topics.TopicDetailView.as_view(),\n name=\"topic-detail\",\n ),\n path(\n \"topics//next-transcribable-asset/\",\n views.assets.redirect_to_next_transcribable_topic_asset,\n name=\"redirect-to-next-transcribable-topic-asset\",\n ),\n path(\n \"topics//next-reviewable-asset/\",\n views.assets.redirect_to_next_reviewable_topic_asset,\n name=\"redirect-to-next-reviewable-topic-asset\",\n ),\n path(\n \"next-transcribable-asset/\",\n views.assets.redirect_to_next_transcribable_asset,\n name=\"redirect-to-next-transcribable-asset\",\n ),\n path(\n \"next-reviewable-asset/\",\n views.assets.redirect_to_next_reviewable_asset,\n name=\"redirect-to-next-reviewable-asset\",\n ),\n path(\"campaigns/\", include(tx_urlpatterns, namespace=\"transcriptions\")),\n path(\n \"reserve-asset//\", views.ajax.reserve_asset, name=\"reserve-asset\"\n ),\n path(\n \"assets//transcriptions/save/\",\n views.ajax.save_transcription,\n name=\"save-transcription\",\n ),\n path(\n \"transcriptions//submit/\",\n views.ajax.submit_transcription,\n name=\"submit-transcription\",\n ),\n path(\n \"transcriptions//review/\",\n views.ajax.review_transcription,\n name=\"review-transcription\",\n ),\n path(\n \"assets//transcriptions/generate-ocr/\",\n views.ajax.generate_ocr_transcription,\n name=\"generate-ocr-transcription\",\n ),\n path(\n \"assets//transcriptions/rollback/\",\n views.ajax.rollback_transcription,\n name=\"rollback-transcription\",\n ),\n path(\n \"assets//transcriptions/rollforward/\",\n views.ajax.rollforward_transcription,\n name=\"rollforward-transcription\",\n ),\n path(\n \"assets//tags/submit/\", views.ajax.submit_tags, name=\"submit-tags\"\n ),\n path(\n \"account/ajax-status/\",\n views.ajax.ajax_session_status,\n name=\"ajax-session-status\",\n ),\n path(\"account/ajax-messages/\", views.ajax.ajax_messages, name=\"ajax-messages\"),\n path(\n \"account/register/\",\n views.accounts.ConcordiaRegistrationView.as_view(),\n name=\"registration_register\",\n ),\n path(\n \"account/login/\",\n views.accounts.ConcordiaLoginView.as_view(),\n name=\"registration_login\",\n ),\n path(\"account/get_pages/\", views.accounts.get_pages, name=\"get_pages\"),\n path(\n \"account/profile/\",\n views.accounts.AccountProfileView.as_view(),\n name=\"user-profile\",\n ),\n path(\n \"account/password_reset/\",\n views.accounts.ConcordiaPasswordResetRequestView.as_view(),\n name=\"password_reset\",\n ),\n path(\n \"account/reset///\",\n views.accounts.ConcordiaPasswordResetConfirmView.as_view(),\n name=\"password_reset_confirm\",\n ),\n path(\"account/\", include(\"django_registration.backends.activation.urls\")),\n path(\"account/\", include(\"django.contrib.auth.urls\")),\n path(\n \"account/email_confirmation//\",\n views.accounts.EmailReconfirmationView.as_view(),\n name=\"email-reconfirmation\",\n ),\n path(\n \"account/delete/\",\n views.accounts.AccountDeletionView.as_view(),\n name=\"account-deletion\",\n ),\n path(\n \".well-known/change-password\", # https://wicg.github.io/change-password-url/\n RedirectView.as_view(pattern_name=\"password_change\"),\n ),\n path(\"admin/\", admin.site.urls),\n # Internal support assists:\n path(\"error/500/\", server_error),\n path(\"error/404/\", page_not_found, {\"exception\": Http404()}),\n path(\"error/429/\", views.rate_limit.ratelimit_view),\n path(\"error/403/\", permission_denied, {\"exception\": HttpResponseForbidden()}),\n path(\"tinymce/\", include(\"tinymce.urls\")),\n path(\"metrics\", MetricsView.as_view(), name=\"prometheus-django-metrics\"),\n path(\"robots.txt\", include(\"robots.urls\")),\n path(\n \"maintenance-mode/off/\",\n views.maintenance_mode.maintenance_mode_off,\n name=\"maintenance_mode_off\",\n ),\n path(\n \"maintenance-mode/on/\",\n views.maintenance_mode.maintenance_mode_on,\n name=\"maintenance_mode_on\",\n ),\n path(\n \"maintenance-mode/frontend/available\",\n views.maintenance_mode.maintenance_mode_frontend_available,\n name=\"maintenance_mode_frontend_available\",\n ),\n path(\n \"maintenance-mode/frontend/unavailable\",\n views.maintenance_mode.maintenance_mode_frontend_unavailable,\n name=\"maintenance_mode_frontend_unavailable\",\n ),\n path(\n \"api/visualization//\",\n views.visualizations.VisualizationDataView.as_view(),\n name=\"visualization\",\n ),\n]\n\nif settings.DEBUG:\n import debug_toolbar\n from django.conf.urls.static import static\n from django.views.generic import TemplateView\n\n from concordia.api import api as concordia_api\n\n urlpatterns = [path(\"__debug__/\", include(debug_toolbar.urls))] + urlpatterns\n\n urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)\n\n urlpatterns += (\n path(\n \"transcription/\",\n TemplateView.as_view(template_name=\"transcriptions/transcription.html\"),\n name=\"transcription\",\n ),\n path(\"api/\", concordia_api.urls, name=\"api\"),\n )\n"
},
{
"path": "concordia/utils/__init__.py",
"content": "from secrets import token_hex\n\nfrom django.contrib.auth.models import User\n\nfrom concordia.logging import ConcordiaLogger\nfrom concordia.templatetags.concordia_media_tags import asset_media_url\n\n__all__ = [\n \"get_anonymous_user\",\n \"request_accepts_json\",\n \"get_or_create_reservation_token\",\n \"get_image_urls_from_asset\",\n]\n\nstructured_logger = ConcordiaLogger.get_logger(__name__)\n\n\ndef get_anonymous_user():\n \"\"\"\n Get the user called \"anonymous\" if it exist. Create the user if it doesn't\n exist This is the default concordia user if someone is working on the site\n without logging in first.\n \"\"\"\n\n try:\n return User.objects.get(username=\"anonymous\")\n except User.DoesNotExist:\n return User.objects.create_user(username=\"anonymous\")\n\n\ndef request_accepts_json(request):\n accept_header = request.headers.get(\"Accept\", \"*/*\")\n\n return \"application/json\" in accept_header\n\n\ndef get_or_create_reservation_token(request):\n # Reservation tokens are 44 characters (22 bytes\n # converted into 44 hex digits) plus the user's\n # database id padded with leading zeroes until it's\n # at least 6 characters long\n if \"reservation_token\" not in request.session:\n request.session[\"reservation_token\"] = token_hex(22)\n user = getattr(request, \"user\", None)\n if user is not None:\n uid = user.id\n if uid is None:\n uid = get_anonymous_user().id\n request.session[\"reservation_token\"] += str(uid).zfill(6)\n structured_logger.info(\n \"Reservation token created.\",\n event_code=\"reservation_token_created\",\n reservation_token=request.session[\"reservation_token\"],\n user=user,\n )\n else:\n structured_logger.info(\n \"Reservation token reused.\",\n event_code=\"reservation_token_reused\",\n reservation_token=request.session[\"reservation_token\"],\n )\n return request.session[\"reservation_token\"]\n\n\ndef get_image_urls_from_asset(asset):\n \"\"\"\n Given an Asset, return a tuple containing the normalized full-size and\n thumbnail-size image URLs\n \"\"\"\n\n image_url = asset_media_url(asset)\n if asset.download_url and \"iiif\" in asset.download_url:\n thumbnail_url = asset.download_url.replace(\n \"http://tile.loc.gov\", \"https://tile.loc.gov\"\n )\n else:\n thumbnail_url = image_url\n\n return image_url, thumbnail_url\n"
},
{
"path": "concordia/utils/celery.py",
"content": "from celery import Task\n\nfrom concordia.celery import app as concordia_celery_app\n\n\ndef get_registered_task(name: str) -> Task:\n \"\"\"\n Retrieve a Celery task by its fully qualified name.\n\n This function looks up a task in the Celery app task registry. It raises a\n RuntimeError if the task is not found. The purpose of this function is to\n provide a usable interface for safely calling a task without importing it\n directly, to avoid issues such as circular imports. This avoids issues with\n `app.send_task`, which ignores settings such as `ALWAYS_EAGER`.\n\n Args:\n name (str): Fully qualified task name, for example\n \"myapp.tasks.my_task\".\n\n Returns:\n Task: The registered Celery task object.\n\n Raises:\n RuntimeError: If the task name is not found in the registry.\n \"\"\"\n try:\n return concordia_celery_app.tasks[name]\n except KeyError as err:\n raise RuntimeError(f\"Task {name} is not registered. Did you typo it?\") from err\n"
},
{
"path": "concordia/utils/constants.py",
"content": "from django.contrib import messages\n\nASSETS_PER_PAGE = 36\nPROJECTS_PER_PAGE = 36\nITEMS_PER_PAGE = 36\nURL_REGEX = r\"http[s]?://\"\n\nMESSAGE_LEVEL_NAMES = dict(\n zip(\n messages.DEFAULT_LEVELS.values(),\n map(str.lower, messages.DEFAULT_LEVELS.keys()),\n strict=False,\n )\n)\n"
},
{
"path": "concordia/utils/next_asset/__init__.py",
"content": "from concordia.logging import ConcordiaLogger\nfrom concordia.models import (\n NextReviewableCampaignAsset,\n NextReviewableTopicAsset,\n NextTranscribableCampaignAsset,\n NextTranscribableTopicAsset,\n)\n\nfrom .reviewable import (\n find_and_order_potential_reviewable_campaign_assets,\n find_and_order_potential_reviewable_topic_assets,\n find_invalid_next_reviewable_campaign_assets,\n find_invalid_next_reviewable_topic_assets,\n find_new_reviewable_campaign_assets,\n find_new_reviewable_topic_assets,\n find_next_reviewable_campaign_asset,\n find_next_reviewable_campaign_assets,\n find_next_reviewable_topic_asset,\n find_next_reviewable_topic_assets,\n find_reviewable_campaign_asset,\n find_reviewable_topic_asset,\n)\nfrom .transcribable import (\n find_and_order_potential_transcribable_campaign_assets,\n find_and_order_potential_transcribable_topic_assets,\n find_invalid_next_transcribable_campaign_assets,\n find_invalid_next_transcribable_topic_assets,\n find_new_transcribable_campaign_assets,\n find_new_transcribable_topic_assets,\n find_next_transcribable_campaign_asset,\n find_next_transcribable_campaign_assets,\n find_next_transcribable_topic_asset,\n find_next_transcribable_topic_assets,\n find_transcribable_campaign_asset,\n find_transcribable_topic_asset,\n)\n\n__all__ = [\n \"find_and_order_potential_transcribable_campaign_assets\",\n \"find_and_order_potential_transcribable_topic_assets\",\n \"find_new_transcribable_campaign_assets\",\n \"find_new_transcribable_topic_assets\",\n \"find_next_transcribable_campaign_asset\",\n \"find_next_transcribable_topic_asset\",\n \"find_next_transcribable_campaign_assets\",\n \"find_next_transcribable_topic_assets\",\n \"find_transcribable_campaign_asset\",\n \"find_transcribable_topic_asset\",\n \"find_and_order_potential_reviewable_campaign_assets\",\n \"find_and_order_potential_reviewable_topic_assets\",\n \"find_new_reviewable_campaign_assets\",\n \"find_new_reviewable_topic_assets\",\n \"find_next_reviewable_campaign_assets\",\n \"find_next_reviewable_topic_assets\",\n \"find_next_reviewable_campaign_asset\",\n \"find_next_reviewable_topic_asset\",\n \"find_reviewable_campaign_asset\",\n \"find_reviewable_topic_asset\",\n \"remove_next_asset_objects\",\n \"find_invalid_next_reviewable_campaign_assets\",\n \"find_invalid_next_reviewable_topic_assets\",\n \"find_invalid_next_transcribable_campaign_assets\",\n \"find_invalid_next_transcribable_topic_assets\",\n]\n\nstructured_logger = ConcordiaLogger.get_logger(__name__)\n\n\ndef remove_next_asset_objects(asset_id):\n \"\"\"\n Remove all cached next asset entries associated with the given asset id.\n\n This function deletes entries from the four next asset tables:\n - NextTranscribableCampaignAsset\n - NextTranscribableTopicAsset\n - NextReviewableCampaignAsset\n - NextReviewableTopicAsset\n\n It is typically used when an asset is no longer valid for caching\n (e.g., after transcription or review status changes, or when a user\n reserves the asset).\n\n Args:\n asset_id (int): The ID of the asset to remove from next-asset tables.\n \"\"\"\n structured_logger.info(\n \"Removing next asset objects\",\n event_code=\"remove_next_asset_objects\",\n asset_id=asset_id,\n )\n NextTranscribableCampaignAsset.objects.filter(asset_id=asset_id).delete()\n NextTranscribableTopicAsset.objects.filter(asset_id=asset_id).delete()\n NextReviewableCampaignAsset.objects.filter(asset_id=asset_id).delete()\n NextReviewableTopicAsset.objects.filter(asset_id=asset_id).delete()\n"
},
{
"path": "concordia/utils/next_asset/reviewable/__init__.py",
"content": "from .campaign import (\n find_and_order_potential_reviewable_campaign_assets,\n find_invalid_next_reviewable_campaign_assets,\n find_new_reviewable_campaign_assets,\n find_next_reviewable_campaign_asset,\n find_next_reviewable_campaign_assets,\n find_reviewable_campaign_asset,\n)\nfrom .topic import (\n find_and_order_potential_reviewable_topic_assets,\n find_invalid_next_reviewable_topic_assets,\n find_new_reviewable_topic_assets,\n find_next_reviewable_topic_asset,\n find_next_reviewable_topic_assets,\n find_reviewable_topic_asset,\n)\n\n__all__ = [\n \"find_new_reviewable_campaign_assets\",\n \"find_next_reviewable_campaign_assets\",\n \"find_reviewable_campaign_asset\",\n \"find_and_order_potential_reviewable_campaign_assets\",\n \"find_next_reviewable_campaign_asset\",\n \"find_and_order_potential_reviewable_topic_assets\",\n \"find_new_reviewable_topic_assets\",\n \"find_next_reviewable_topic_asset\",\n \"find_next_reviewable_topic_assets\",\n \"find_reviewable_topic_asset\",\n \"find_invalid_next_reviewable_campaign_assets\",\n \"find_invalid_next_reviewable_topic_assets\",\n]\n"
},
{
"path": "concordia/utils/next_asset/reviewable/campaign.py",
"content": "from typing import Dict\n\nfrom django.contrib.auth.models import User\nfrom django.db import transaction\nfrom django.db.models import Case, IntegerField, Q, QuerySet, Subquery, Value, When\n\nfrom concordia import models as concordia_models\nfrom concordia.logging import ConcordiaLogger\nfrom concordia.utils.celery import get_registered_task\n\nstructured_logger = ConcordiaLogger.get_logger(__name__)\n\n\ndef _reserved_asset_ids_subq(\n campaign: concordia_models.Campaign,\n) -> \"QuerySet[Dict[str, int]]\":\n \"\"\"\n Return a subquery of reserved asset identifiers for a campaign.\n\n Behavior:\n Produces a subquery suitable for use with `Subquery(...)` and\n `exclude(pk__in=...)` clauses to filter out assets that currently have\n an active reservation.\n\n Args:\n campaign (concordia_models.Campaign): Campaign whose reserved\n assets should be excluded.\n\n Returns:\n QuerySet[Dict[str, int]]: A queryset of dictionaries with a single key\n \"asset_id\" corresponding to reserved assets.\n \"\"\"\n return concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__campaign=campaign\n ).values(\"asset_id\")\n\n\ndef _eligible_reviewable_base_qs(\n campaign: concordia_models.Campaign,\n user: User | None = None,\n) -> \"QuerySet[concordia_models.Asset]\":\n \"\"\"\n Build the base queryset of reviewable assets for a campaign.\n\n Behavior:\n Restricts to published projects, items, and assets, and to assets whose\n transcription status is `SUBMITTED`. Optionally excludes assets\n transcribed by the supplied user.\n\n Args:\n campaign (concordia_models.Campaign): Campaign scope for filtering.\n user (User | None): If provided, exclude assets transcribed by this user.\n\n Returns:\n QuerySet[concordia_models.Asset]: Reviewable assets, with `item` and\n `item__project` selected via `select_related`.\n \"\"\"\n qs = concordia_models.Asset.objects.filter(\n campaign_id=campaign.id,\n item__project__published=True,\n item__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,\n ).select_related(\"item\", \"item__project\")\n if user:\n qs = qs.exclude(transcription__user=user.id)\n return qs\n\n\ndef _next_seq_after(pk: int | None) -> int | None:\n \"\"\"\n Resolve the sequence number for a given asset primary key.\n\n Behavior:\n Convenience utility for ordering logic when advancing within a series\n of assets.\n\n Args:\n pk (int | None): Asset primary key whose sequence to resolve.\n\n Returns:\n int | None: The asset's sequence number, or None if `pk` is falsy\n or the asset does not exist.\n \"\"\"\n if not pk:\n return None\n return (\n concordia_models.Asset.objects.filter(pk=pk)\n .values_list(\"sequence\", flat=True)\n .first()\n )\n\n\n@transaction.atomic\ndef _find_reviewable_in_item(\n campaign: concordia_models.Campaign,\n user: User,\n *,\n item_id: str,\n after_asset_pk: int | None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Select the next reviewable asset within the same item.\n\n Behavior:\n Attempts a short-circuit within the user's current item to provide a\n locally contiguous review flow.\n\n Eligibility:\n - Asset, Item, and Project are published.\n - Asset transcription status is `SUBMITTED`.\n - Asset is not reserved.\n - Asset was not transcribed by the current user.\n\n Ordering:\n - If `after_asset_pk` refers to an asset in the same item and campaign,\n select the earliest asset whose (sequence, id) is strictly greater\n than the current asset's pair.\n - Otherwise, select the earliest eligible by (sequence, id).\n\n Args:\n campaign (concordia_models.Campaign): Campaign scope.\n user (User): Current user; used to exclude their own work.\n item_id (str): Identifier of the item to stay within.\n after_asset_pk (int | None): Asset primary key to advance from.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or\n None if no match is available.\n \"\"\"\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__item__item_id=item_id,\n asset__item__project__campaign=campaign,\n ).values(\"asset_id\")\n\n eligible = (\n concordia_models.Asset.objects.filter(\n item__item_id=item_id,\n item__project__campaign=campaign,\n item__project__published=True,\n item__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,\n )\n .exclude(pk__in=Subquery(reserved_asset_ids))\n .exclude(transcription__user=user.id)\n )\n\n seq_gt_filter = None\n if after_asset_pk is not None:\n try:\n current = (\n concordia_models.Asset.objects.only(\"id\", \"sequence\", \"item_id\", \"item\")\n .select_related(\"item\")\n .get(pk=after_asset_pk)\n )\n if (\n current.item.item_id == item_id\n and current.item.project.campaign_id == campaign.id\n ):\n seq_gt_filter = Q(sequence__gt=current.sequence) | (\n Q(sequence=current.sequence) & Q(id__gt=after_asset_pk)\n )\n except concordia_models.Asset.DoesNotExist:\n pass\n\n if seq_gt_filter is not None:\n eligible = eligible.filter(seq_gt_filter)\n\n asset = (\n eligible.select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .order_by(\"sequence\", \"id\")\n .first()\n )\n\n structured_logger.debug(\n \"Item short-circuit (campaign reviewable) resolved.\",\n event_code=\"reviewable_item_short_circuit_campaign\",\n campaign=campaign,\n item_id=item_id,\n after_asset_pk=after_asset_pk,\n chosen_asset_id=getattr(asset, \"id\", None),\n )\n return asset\n\n\n@transaction.atomic\ndef _find_reviewable_in_project(\n campaign: concordia_models.Campaign,\n user: User,\n *,\n project_slug: str,\n after_asset_pk: int | None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Select the first eligible reviewable asset within the same project.\n\n Behavior:\n Short-circuit when staying within a project. Sequence is per item,\n so this returns the first eligible asset, not strictly \"after\" a given asset.\n\n Eligibility:\n - Same campaign and project.\n - Asset, Item, and Project are published.\n - Asset transcription status is `SUBMITTED`.\n - Asset is not reserved.\n - Asset was not transcribed by the current user.\n\n Ordering:\n Deterministic by (item__item_id, sequence, id).\n\n Args:\n campaign (concordia_models.Campaign): Campaign scope.\n user (User): Current user; used to exclude their own work.\n project_slug (str): Slug of the project to stay within.\n after_asset_pk (int | None): Present for parity with the item\n variant; not used for ordering here.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or\n None if no match is available.\n \"\"\"\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__item__project__slug=project_slug,\n asset__item__project__campaign=campaign,\n ).values(\"asset_id\")\n\n eligible = (\n concordia_models.Asset.objects.filter(\n item__project__campaign=campaign,\n item__project__slug=project_slug,\n item__project__published=True,\n item__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,\n )\n .exclude(pk__in=Subquery(reserved_asset_ids))\n .exclude(transcription__user=user.id)\n .select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .order_by(\"item__item_id\", \"sequence\", \"id\")\n .first()\n )\n\n structured_logger.debug(\n \"Project short-circuit (campaign reviewable) resolved.\",\n event_code=\"reviewable_project_short_circuit_campaign\",\n campaign=campaign,\n project_slug=project_slug,\n after_asset_pk=after_asset_pk,\n chosen_asset_id=getattr(eligible, \"id\", None),\n )\n return eligible\n\n\ndef find_new_reviewable_campaign_assets(\n campaign: concordia_models.Campaign,\n user: User | None = None,\n) -> \"QuerySet[concordia_models.Asset]\":\n \"\"\"\n Return assets in a campaign that are eligible to be added to the cache.\n\n Behavior:\n Builds the candidate set for the NextReviewableCampaignAsset cache by\n excluding assets that are not `SUBMITTED`, assets already reserved, and\n assets already present in the cache. Optionally excludes assets\n transcribed by the provided user.\n\n Args:\n campaign (concordia_models.Campaign): Campaign to filter by.\n user (User | None): If provided, exclude assets transcribed by this user.\n\n Returns:\n QuerySet[concordia_models.Asset]: Eligible assets ordered by sequence.\n \"\"\"\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__campaign=campaign\n ).values(\"asset_id\")\n next_asset_ids = concordia_models.NextReviewableCampaignAsset.objects.filter(\n campaign=campaign\n ).values(\"asset_id\")\n\n queryset = (\n concordia_models.Asset.objects.filter(\n campaign_id=campaign.id,\n item__project__published=True,\n item__published=True,\n published=True,\n )\n .filter(transcription_status=concordia_models.TranscriptionStatus.SUBMITTED)\n .exclude(pk__in=Subquery(reserved_asset_ids))\n .exclude(pk__in=Subquery(next_asset_ids))\n .order_by(\"sequence\")\n )\n if user:\n queryset = queryset.exclude(transcription__user=user.id)\n return queryset\n\n\ndef find_next_reviewable_campaign_assets(\n campaign: concordia_models.Campaign,\n user: User,\n) -> \"QuerySet[concordia_models.NextReviewableCampaignAsset]\":\n \"\"\"\n Return cached reviewable assets in a campaign not transcribed by the user.\n\n Behavior:\n Reads from the NextReviewableCampaignAsset cache table and filters out\n assets where the requesting user appears in transcriber_ids.\n\n Args:\n campaign (concordia_models.Campaign): Campaign to retrieve cached assets from.\n user (User): Requesting user.\n\n Returns:\n QuerySet[concordia_models.NextReviewableCampaignAsset]: Cached candidate rows\n for the given user.\n \"\"\"\n return concordia_models.NextReviewableCampaignAsset.objects.filter(\n campaign=campaign\n ).exclude(transcriber_ids__contains=[user.id])\n\n\n@transaction.atomic\ndef find_reviewable_campaign_asset(\n campaign: concordia_models.Campaign,\n user: User,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Retrieve a single reviewable asset for a user from a campaign.\n\n Behavior:\n First attempts to select a cached asset from NextReviewableCampaignAsset.\n If none is available, falls back to a direct query over Asset and\n triggers a background task to replenish the cache.\n\n Concurrency:\n Uses `select_for_update(skip_locked=True, of=(\"self\",))` so only the\n Asset row is locked and concurrent consumers skip locked rows.\n\n Args:\n campaign (concordia_models.Campaign): Campaign to search within.\n user (User): Requesting user; their own transcriptions are excluded.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or None if unavailable.\n \"\"\"\n next_asset = (\n find_next_reviewable_campaign_assets(campaign, user)\n .select_for_update(skip_locked=True, of=(\"self\",))\n .values_list(\"asset_id\", flat=True)\n .first()\n )\n\n spawn_task = False\n if next_asset:\n asset_query = concordia_models.Asset.objects.filter(id=next_asset)\n else:\n # No asset in the NextReviewableCampaignAsset table for this campaign\n # and user, so fallback to manually finding one\n structured_logger.debug(\n \"No cached assets available, falling back to manual lookup\",\n event_code=\"reviewable_fallback_manual_lookup\",\n campaign=campaign,\n user=user,\n )\n spawn_task = True\n asset_query = find_new_reviewable_campaign_assets(campaign, user)\n\n # select_for_update(of=(\"self\",)) causes the row locking only to\n # apply to the Asset table, rather than also locking joined item table\n asset = (\n asset_query.select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n\n if spawn_task:\n # Spawn a task to populate the table for this campaign\n # We wait to do this until after getting an asset because otherwise there's\n # a chance all valid assets get grabbed by the task and our query will return\n # nothing\n structured_logger.debug(\n \"Spawned background task to populate cache\",\n event_code=\"reviewable_cache_population_triggered\",\n campaign=campaign,\n user=user,\n )\n populate_task = get_registered_task(\n \"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_campaign\"\n )\n populate_task.delay(campaign.id)\n\n return asset\n\n\ndef find_and_order_potential_reviewable_campaign_assets(\n campaign: concordia_models.Campaign,\n user: User,\n project_slug: str,\n item_id: str,\n asset_pk: int | None,\n) -> \"QuerySet[concordia_models.NextReviewableCampaignAsset]\":\n \"\"\"\n Retrieve and prioritize cached reviewable assets for proximity.\n\n Behavior:\n Orders cached candidates from NextReviewableCampaignAsset to prefer\n continuity with the user's current location.\n\n Annotations added to each row (transient fields):\n - next_asset (int): 1 if the candidate's asset_id is greater than\n asset_pk, else 0.\n - same_project (int): 1 if the candidate shares the given\n project_slug, else 0.\n - same_item (int): 1 if the candidate shares the given item_id, else 0.\n\n Prioritization (descending on the following keys, then ascending by sequence):\n - next_asset\n - same_project\n - same_item\n - sequence\n\n Args:\n campaign (concordia_models.Campaign): Campaign to filter by.\n user (User): Requesting user.\n project_slug (str): Slug of the user's current project.\n item_id (str): Identifier of the user's current item.\n asset_pk (int | None): Identifier of the current asset, if any.\n\n Returns:\n QuerySet[concordia_models.NextReviewableCampaignAsset]: Prioritized\n cached candidates.\n \"\"\"\n potential_next_assets = find_next_reviewable_campaign_assets(campaign, user)\n\n # We'll favor assets which are in the same item or project as the original:\n next_case = (\n Case(\n When(asset_id__gt=asset_pk, then=1),\n default=0,\n output_field=IntegerField(),\n )\n if asset_pk is not None\n else Value(0, output_field=IntegerField())\n )\n\n potential_next_assets = potential_next_assets.annotate(\n same_project=Case(\n When(project_slug=project_slug, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n same_item=Case(\n When(item_item_id=item_id, then=1), default=0, output_field=IntegerField()\n ),\n next_asset=next_case,\n ).order_by(\"-next_asset\", \"-same_project\", \"-same_item\", \"sequence\")\n\n return potential_next_assets\n\n\n@transaction.atomic\ndef find_next_reviewable_campaign_asset(\n campaign: concordia_models.Campaign,\n user: User,\n project_slug: str,\n item_id: str,\n original_asset_id: int | None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Retrieve the next best reviewable asset for a user within a campaign.\n\n Strategy:\n 1. If `item_id` is provided, try a same-item short-circuit that advances\n by (sequence, id) relative to `original_asset_id`.\n 2. Else, if `project_slug` is provided, select the first eligible asset\n within that project (short-circuit).\n 3. Else, prioritize cached candidates, and if none are suitable, fall\n back to computing from Asset and trigger cache population.\n\n Concurrency:\n Uses `select_for_update(skip_locked=True, of=(\"self\",))` to avoid\n double-assignments across concurrent consumers.\n\n Args:\n campaign (concordia_models.Campaign): Campaign to search within.\n user (User): Requesting user.\n project_slug (str): Slug of the user's current project.\n item_id (str): Identifier of the user's current item.\n original_asset_id (int | None): Identifier of the asset just reviewed.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or None if unavailable.\n \"\"\"\n try:\n after_pk = int(original_asset_id) if original_asset_id else None\n except (TypeError, ValueError):\n after_pk = None\n\n # Short-circuit: same item\n if item_id:\n asset = _find_reviewable_in_item(\n campaign, user, item_id=item_id, after_asset_pk=after_pk\n )\n if asset:\n return asset\n\n # Short-circuit: same project\n if project_slug:\n asset = _find_reviewable_in_project(\n campaign, user, project_slug=project_slug, after_asset_pk=after_pk\n )\n if asset:\n return asset\n\n # cache-backed selection, then manual fallback\n potential_next_assets = find_and_order_potential_reviewable_campaign_assets(\n campaign, user, project_slug, item_id, after_pk\n )\n\n asset_id = (\n potential_next_assets.select_for_update(skip_locked=True, of=(\"self\",))\n .values_list(\"asset_id\", flat=True)\n .first()\n )\n\n spawn_task = False\n if asset_id:\n asset_query = concordia_models.Asset.objects.filter(id=asset_id)\n else:\n # Since we had no potential next assets in the caching table, we have to check\n # the asset table directly.\n structured_logger.debug(\n \"No cached assets matched, falling back to manual lookup\",\n event_code=\"reviewable_next_fallback_manual\",\n campaign=campaign,\n user=user,\n )\n spawn_task = True\n asset_query = find_new_reviewable_campaign_assets(campaign, user)\n\n next_case = (\n Case(\n When(id__gt=after_pk, then=1),\n default=0,\n output_field=IntegerField(),\n )\n if after_pk is not None\n else Value(0, output_field=IntegerField())\n )\n\n asset_query = asset_query.annotate(\n same_project=Case(\n When(item__project__slug=project_slug, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n same_item=Case(\n When(item__item_id=item_id, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n next_asset=next_case,\n ).order_by(\"-next_asset\", \"-same_project\", \"-same_item\", \"sequence\")\n\n asset = (\n asset_query.select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n\n if spawn_task:\n # Spawn a task to populate the table for this campaign\n # We wait to do this until after getting an asset because otherwise there's\n # a chance all valid assets get grabbed by the task and our query will return\n # nothing\n structured_logger.debug(\n \"Spawned background task to populate cache\",\n event_code=\"reviewable_next_cache_population\",\n campaign=campaign,\n user=user,\n )\n populate_task = get_registered_task(\n \"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_campaign\"\n )\n populate_task.delay(campaign.id)\n\n return asset\n\n\ndef find_invalid_next_reviewable_campaign_assets(\n campaign_id: int,\n) -> \"QuerySet[concordia_models.NextReviewableCampaignAsset]\":\n \"\"\"\n Return cache rows that are invalid for review for a given campaign.\n\n Behavior:\n Identifies NextReviewableCampaignAsset rows that are no longer valid\n because the underlying asset is not `SUBMITTED` or because the asset is\n currently reserved.\n\n Args:\n campaign_id (int): Identifier of the campaign.\n\n Returns:\n QuerySet[concordia_models.NextReviewableCampaignAsset]: Distinct\n invalid cache rows.\n \"\"\"\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__campaign_id=campaign_id\n ).values(\"asset_id\")\n\n status_filtered = concordia_models.NextReviewableCampaignAsset.objects.exclude(\n asset__transcription_status=concordia_models.TranscriptionStatus.SUBMITTED\n ).filter(campaign_id=campaign_id)\n\n reserved_filtered = concordia_models.NextReviewableCampaignAsset.objects.filter(\n campaign_id=campaign_id, asset_id__in=Subquery(reserved_asset_ids)\n )\n\n return (status_filtered | reserved_filtered).distinct()\n"
},
{
"path": "concordia/utils/next_asset/reviewable/topic.py",
"content": "from typing import Dict\n\nfrom django.contrib.auth.models import User\nfrom django.db import transaction\nfrom django.db.models import Case, IntegerField, Q, QuerySet, Subquery, Value, When\n\nfrom concordia import models as concordia_models\nfrom concordia.logging import ConcordiaLogger\nfrom concordia.utils.celery import get_registered_task\n\nstructured_logger = ConcordiaLogger.get_logger(__name__)\n\n\ndef _reserved_asset_ids_subq() -> \"QuerySet[Dict[str, int]]\":\n \"\"\"\n Return a subquery of reserved asset identifiers.\n\n Behavior:\n Produces a subquery suitable for use with `Subquery(...)` and\n `exclude(pk__in=...)` to filter out assets that currently have an\n active reservation. This is not filtered to the topic to avoid\n additional joins.\n\n Returns:\n QuerySet[Dict[str, int]]: A queryset of dictionaries with a single key\n \"asset_id\" corresponding to reserved assets.\n \"\"\"\n return concordia_models.AssetTranscriptionReservation.objects.values(\"asset_id\")\n\n\ndef _eligible_reviewable_base_qs(\n topic: concordia_models.Topic,\n user: User | None = None,\n) -> \"QuerySet[concordia_models.Asset]\":\n \"\"\"\n Build the base queryset of reviewable assets for a topic.\n\n Behavior:\n Restricts to published projects, items, and assets, and to assets whose\n transcription status is `SUBMITTED`. Optionally excludes assets\n transcribed by the supplied user.\n\n Args:\n topic (concordia_models.Topic): Topic scope for filtering.\n user (User | None): If provided, exclude assets transcribed by this user.\n\n Returns:\n QuerySet[concordia_models.Asset]: Reviewable assets, with `item` and\n `item__project` selected via `select_related`.\n \"\"\"\n qs = concordia_models.Asset.objects.filter(\n item__project__topics=topic.id,\n item__project__published=True,\n item__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,\n ).select_related(\"item\", \"item__project\")\n if user:\n qs = qs.exclude(transcription__user=user.id)\n return qs\n\n\ndef _next_seq_after(pk: int | None) -> int | None:\n \"\"\"\n Resolve the sequence number for a given asset primary key.\n\n Behavior:\n Convenience utility for ordering logic when advancing within a series\n of assets.\n\n Args:\n pk (int | None): Asset primary key whose sequence to resolve.\n\n Returns:\n int | None: The asset's sequence number, or None if `pk` is falsy\n or the asset does not exist.\n \"\"\"\n if not pk:\n return None\n return (\n concordia_models.Asset.objects.filter(pk=pk)\n .values_list(\"sequence\", flat=True)\n .first()\n )\n\n\n@transaction.atomic\ndef _find_reviewable_in_item(\n topic: concordia_models.Topic,\n user: User,\n *,\n item_id: str,\n after_asset_pk: int | None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Select the next reviewable asset within the same item.\n\n Behavior:\n Attempts a short-circuit within the user's current item to provide a\n locally contiguous review flow.\n\n Eligibility:\n - Asset, Item, and Project are published.\n - Asset transcription status is `SUBMITTED`.\n - Asset is not reserved.\n - Asset was not transcribed by the current user.\n\n Ordering:\n - If `after_asset_pk` refers to an asset in the same item whose project\n is in `topic`, select the earliest asset whose (sequence, id) is\n strictly greater than the current asset's pair.\n - Otherwise, select the earliest eligible by (sequence, id).\n\n Args:\n topic (concordia_models.Topic): Topic scope.\n user (User): Current user; used to exclude their own work.\n item_id (str): Identifier of the item to stay within.\n after_asset_pk (int | None): Asset primary key to advance from.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or\n None if no match is available.\n \"\"\"\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__item__item_id=item_id,\n asset__item__project__topics=topic,\n ).values(\"asset_id\")\n\n eligible = (\n concordia_models.Asset.objects.filter(\n item__item_id=item_id,\n item__project__topics=topic,\n item__project__published=True,\n item__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,\n )\n .exclude(pk__in=Subquery(reserved_asset_ids))\n .exclude(transcription__user=user.id)\n )\n\n seq_gt_filter = None\n if after_asset_pk is not None:\n try:\n current = (\n concordia_models.Asset.objects.only(\"id\", \"sequence\", \"item_id\", \"item\")\n .select_related(\"item\", \"item__project\")\n .get(pk=after_asset_pk)\n )\n if (\n current.item.item_id == item_id\n and current.item.project.topics.filter(pk=topic.pk).exists()\n ):\n seq_gt_filter = Q(sequence__gt=current.sequence) | (\n Q(sequence=current.sequence) & Q(id__gt=after_asset_pk)\n )\n except concordia_models.Asset.DoesNotExist:\n pass\n\n if seq_gt_filter is not None:\n eligible = eligible.filter(seq_gt_filter)\n\n asset = (\n eligible.select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .order_by(\"sequence\", \"id\")\n .first()\n )\n\n structured_logger.debug(\n \"Item short-circuit (topic reviewable) resolved.\",\n event_code=\"reviewable_item_short_circuit_topic\",\n topic=topic,\n item_id=item_id,\n after_asset_pk=after_asset_pk,\n chosen_asset_id=getattr(asset, \"id\", None),\n )\n return asset\n\n\n@transaction.atomic\ndef _find_reviewable_in_project(\n topic: concordia_models.Topic,\n user: User,\n *,\n project_slug: str,\n after_asset_pk: int | None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Select the first eligible reviewable asset within the same project.\n\n Behavior:\n Short-circuit when staying within a project. Sequence is per item,\n so this returns the first eligible asset, not strictly \"after\" a given\n asset.\n\n Eligibility:\n - Same topic and project.\n - Asset, Item, and Project are published.\n - Asset transcription status is `SUBMITTED`.\n - Asset is not reserved.\n - Asset was not transcribed by the current user.\n\n Ordering:\n Deterministic by (item__item_id, sequence, id).\n\n Args:\n topic (concordia_models.Topic): Topic scope.\n user (User): Current user; used to exclude their own work.\n project_slug (str): Slug of the project to stay within.\n after_asset_pk (int | None): Present for parity with the item\n variant; not used for ordering here.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or\n None if no match is available.\n \"\"\"\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__item__project__slug=project_slug,\n asset__item__project__topics=topic,\n ).values(\"asset_id\")\n\n eligible = (\n concordia_models.Asset.objects.filter(\n item__project__topics=topic,\n item__project__slug=project_slug,\n item__project__published=True,\n item__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,\n )\n .exclude(pk__in=Subquery(reserved_asset_ids))\n .exclude(transcription__user=user.id)\n .select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .order_by(\"item__item_id\", \"sequence\", \"id\")\n .first()\n )\n\n structured_logger.debug(\n \"Project short-circuit (topic reviewable) resolved.\",\n event_code=\"reviewable_project_short_circuit_topic\",\n topic=topic,\n project_slug=project_slug,\n after_asset_pk=after_asset_pk,\n chosen_asset_id=getattr(eligible, \"id\", None),\n )\n return eligible\n\n\ndef find_new_reviewable_topic_assets(\n topic: concordia_models.Topic,\n user: User | None = None,\n) -> \"QuerySet[concordia_models.Asset]\":\n \"\"\"\n Return assets in a topic that are eligible to be added to the cache.\n\n Behavior:\n Builds the candidate set for the `NextReviewableTopicAsset` cache by\n excluding assets that are not `SUBMITTED`, assets already reserved, and\n assets already present in the cache. Optionally excludes assets\n transcribed by the provided user.\n\n Args:\n topic (concordia_models.Topic): Topic to filter by.\n user (User | None): If provided, exclude assets transcribed by this user.\n\n Returns:\n QuerySet[concordia_models.Asset]: Eligible assets ordered by sequence.\n \"\"\"\n # Filtering this to the topic would be more costly than just getting all ids\n # in most cases because it requires joining the asset table to the item table to\n # the project table to the topic table.\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(\n \"asset_id\"\n )\n next_asset_ids = concordia_models.NextReviewableTopicAsset.objects.filter(\n topic=topic\n ).values(\"asset_id\")\n\n queryset = (\n concordia_models.Asset.objects.filter(\n item__project__topics=topic.id,\n item__project__published=True,\n item__published=True,\n published=True,\n )\n .filter(transcription_status=concordia_models.TranscriptionStatus.SUBMITTED)\n .exclude(pk__in=Subquery(reserved_asset_ids))\n .exclude(pk__in=Subquery(next_asset_ids))\n .order_by(\"sequence\")\n )\n if user:\n queryset = queryset.exclude(transcription__user=user.id)\n return queryset\n\n\ndef find_next_reviewable_topic_assets(\n topic: concordia_models.Topic,\n user: User,\n) -> \"QuerySet[concordia_models.NextReviewableTopicAsset]\":\n \"\"\"\n Return cached reviewable assets in a topic not transcribed by the user.\n\n Behavior:\n Reads from the `NextReviewableTopicAsset` cache table and filters out\n assets where the requesting user appears in `transcriber_ids`.\n\n Args:\n topic (concordia_models.Topic): Topic to retrieve cached assets from.\n user (User): Requesting user.\n\n Returns:\n QuerySet[concordia_models.NextReviewableTopicAsset]: Cached candidate rows\n for the given user.\n \"\"\"\n return concordia_models.NextReviewableTopicAsset.objects.filter(\n topic=topic\n ).exclude(transcriber_ids__contains=[user.id])\n\n\n@transaction.atomic\ndef find_reviewable_topic_asset(\n topic: concordia_models.Topic,\n user: User,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Retrieve a single reviewable asset for a user from a topic.\n\n Behavior:\n First attempts to select a cached asset from `NextReviewableTopicAsset`.\n If none is available, falls back to a direct query over `Asset` and\n triggers a background task to replenish the cache.\n\n Concurrency:\n Uses `select_for_update(skip_locked=True, of=(\"self\",))` so only the\n `Asset` row is locked and concurrent consumers skip locked rows.\n\n Args:\n topic (concordia_models.Topic): Topic to search within.\n user (User): Requesting user; their own transcriptions are excluded.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or None\n if unavailable.\n \"\"\"\n next_asset = (\n find_next_reviewable_topic_assets(topic, user)\n .select_for_update(skip_locked=True, of=(\"self\",))\n .values_list(\"asset_id\", flat=True)\n .first()\n )\n\n spawn_task = False\n if next_asset:\n asset_query = concordia_models.Asset.objects.filter(id=next_asset)\n else:\n # No asset in the NextReviewableTopicAsset table for this topic,\n # so fallback to manually finding one\n structured_logger.debug(\n \"No cached assets available, falling back to manual lookup\",\n event_code=\"reviewable_fallback_manual_lookup\",\n topic=topic,\n user=user,\n )\n spawn_task = True\n asset_query = find_new_reviewable_topic_assets(topic, user)\n\n # select_for_update(of=(\"self\",)) causes the row locking only to\n # apply to the Asset table, rather than also locking joined item table\n asset = (\n asset_query.select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n\n if spawn_task:\n # Spawn a task to populate the table for this topic\n # We wait to do this until after getting an asset because otherwise there's a\n # a chance all valid assets get grabbed by the task and our query will return\n # nothing\n structured_logger.debug(\n \"Spawned background task to populate cache\",\n event_code=\"reviewable_cache_population_triggered\",\n topic=topic,\n user=user,\n )\n populate_task = get_registered_task(\n \"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_topic\"\n )\n populate_task.delay(topic.id)\n\n return asset\n\n\ndef find_and_order_potential_reviewable_topic_assets(\n topic: concordia_models.Topic,\n user: User,\n project_slug: str,\n item_id: str,\n asset_pk: int | None,\n) -> \"QuerySet[concordia_models.NextReviewableTopicAsset]\":\n \"\"\"\n Retrieve and prioritize cached reviewable assets for proximity.\n\n Behavior:\n Orders cached candidates from `NextReviewableTopicAsset` to prefer\n continuity with the user's current location.\n\n Annotations added to each row (transient fields):\n - next_asset (int): 1 if the candidate's asset_id is greater than\n asset_pk, else 0.\n - same_project (int): 1 if the candidate shares the given\n project_slug, else 0.\n - same_item (int): 1 if the candidate shares the given item_id, else 0.\n\n Prioritization (descending on the following keys, then ascending by sequence):\n - next_asset\n - same_project\n - same_item\n - sequence\n\n Args:\n topic (concordia_models.Topic): Topic to filter by.\n user (User): Requesting user.\n project_slug (str): Slug of the user's current project.\n item_id (str): Identifier of the user's current item.\n asset_pk (int | None): Identifier of the current asset, if any.\n\n Returns:\n QuerySet[concordia_models.NextReviewableTopicAsset]: Prioritized\n cached candidates.\n \"\"\"\n potential_next_assets = find_next_reviewable_topic_assets(topic, user)\n\n # Handle None safely for the \"next\" signal\n next_case = (\n Case(\n When(asset_id__gt=asset_pk, then=1),\n default=0,\n output_field=IntegerField(),\n )\n if asset_pk is not None\n else Value(0, output_field=IntegerField())\n )\n\n # We'll favor assets which are in the same item or project as the original:\n potential_next_assets = potential_next_assets.annotate(\n same_project=Case(\n When(project_slug=project_slug, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n same_item=Case(\n When(item_item_id=item_id, then=1), default=0, output_field=IntegerField()\n ),\n next_asset=next_case,\n ).order_by(\"-next_asset\", \"-same_project\", \"-same_item\", \"sequence\")\n\n return potential_next_assets\n\n\n@transaction.atomic\ndef find_next_reviewable_topic_asset(\n topic: concordia_models.Topic,\n user: User,\n project_slug: str,\n item_id: str,\n original_asset_id: int | None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Retrieve the next best reviewable asset for a user within a topic.\n\n Strategy:\n 1. If `item_id` is provided, try a same-item short-circuit that advances\n by (sequence, id) relative to `original_asset_id`.\n 2. Else, if `project_slug` is provided, select the first eligible asset\n within that project (short-circuit).\n 3. Else, prioritize cached candidates, and if none are suitable, fall\n back to computing from `Asset` and trigger cache population.\n\n Concurrency:\n Uses `select_for_update(skip_locked=True, of=(\"self\",))` to avoid\n double-assignments across concurrent consumers.\n\n Args:\n topic (concordia_models.Topic): Topic to search within.\n user (User): Requesting user.\n project_slug (str): Slug of the user's current project.\n item_id (str): Identifier of the user's current item.\n original_asset_id (int | None): Identifier of the asset just reviewed.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or None if\n unavailable.\n \"\"\"\n # Normalize the \"after\" reference\n try:\n after_pk = int(original_asset_id) if original_asset_id else None\n except (TypeError, ValueError):\n after_pk = None\n\n # Short-circuit: same item\n if item_id:\n asset = _find_reviewable_in_item(\n topic, user, item_id=item_id, after_asset_pk=after_pk\n )\n if asset:\n return asset\n\n # Short-circuit: same project\n if project_slug:\n asset = _find_reviewable_in_project(\n topic, user, project_slug=project_slug, after_asset_pk=after_pk\n )\n if asset:\n return asset\n\n # Cache-backed selection, then manual fallback\n potential_next_assets = find_and_order_potential_reviewable_topic_assets(\n topic, user, project_slug, item_id, after_pk\n )\n asset_id = (\n potential_next_assets.select_for_update(skip_locked=True, of=(\"self\",))\n .values_list(\"asset_id\", flat=True)\n .first()\n )\n\n spawn_task = False\n if asset_id:\n asset_query = concordia_models.Asset.objects.filter(id=asset_id)\n else:\n # Since we had no potential next assets in the caching table, we have to check\n # the asset table directly.\n structured_logger.debug(\n \"No cached assets matched, falling back to manual lookup\",\n event_code=\"reviewable_next_fallback_manual\",\n topic=topic,\n user=user,\n )\n spawn_task = True\n asset_query = find_new_reviewable_topic_assets(topic, user)\n\n next_case = (\n Case(\n When(id__gt=after_pk, then=1),\n default=0,\n output_field=IntegerField(),\n )\n if after_pk is not None\n else Value(0, output_field=IntegerField())\n )\n\n asset_query = asset_query.annotate(\n same_project=Case(\n When(item__project__slug=project_slug, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n same_item=Case(\n When(item__item_id=item_id, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n next_asset=next_case,\n ).order_by(\"-next_asset\", \"-same_project\", \"-same_item\", \"sequence\")\n\n asset = (\n asset_query.select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n\n if spawn_task:\n # Spawn a task to populate the table for this topic\n # We wait to do this until after getting an asset because otherwise there's\n # a chance all valid assets get grabbed by the task and our query will return\n # nothing\n structured_logger.debug(\n \"Spawned background task to populate cache\",\n event_code=\"reviewable_next_cache_population\",\n topic=topic,\n user=user,\n )\n populate_task = get_registered_task(\n \"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_topic\"\n )\n populate_task.delay(topic.id)\n\n return asset\n\n\ndef find_invalid_next_reviewable_topic_assets(\n topic_id: int,\n) -> \"QuerySet[concordia_models.NextReviewableTopicAsset]\":\n \"\"\"\n Return cache rows that are invalid for review for a given topic.\n\n Behavior:\n Identifies `NextReviewableTopicAsset` rows that are no longer valid\n because the underlying asset is not `SUBMITTED` or because the asset is\n currently reserved.\n\n Args:\n topic_id (int): Identifier of the topic.\n\n Returns:\n QuerySet[concordia_models.NextReviewableTopicAsset]: Distinct invalid\n cache rows.\n \"\"\"\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__item__project__topics=topic_id\n ).values(\"asset_id\")\n\n status_filtered = concordia_models.NextReviewableTopicAsset.objects.exclude(\n asset__transcription_status=concordia_models.TranscriptionStatus.SUBMITTED\n ).filter(topic_id=topic_id)\n\n reserved_filtered = concordia_models.NextReviewableTopicAsset.objects.filter(\n topic_id=topic_id, asset_id__in=Subquery(reserved_asset_ids)\n )\n\n return (status_filtered | reserved_filtered).distinct()\n"
},
{
"path": "concordia/utils/next_asset/transcribable/__init__.py",
"content": "from .campaign import (\n find_and_order_potential_transcribable_campaign_assets,\n find_invalid_next_transcribable_campaign_assets,\n find_new_transcribable_campaign_assets,\n find_next_transcribable_campaign_asset,\n find_next_transcribable_campaign_assets,\n find_transcribable_campaign_asset,\n)\nfrom .topic import (\n find_and_order_potential_transcribable_topic_assets,\n find_invalid_next_transcribable_topic_assets,\n find_new_transcribable_topic_assets,\n find_next_transcribable_topic_asset,\n find_next_transcribable_topic_assets,\n find_transcribable_topic_asset,\n)\n\n__all__ = [\n \"find_new_transcribable_campaign_assets\",\n \"find_next_transcribable_campaign_assets\",\n \"find_transcribable_campaign_asset\",\n \"find_and_order_potential_transcribable_campaign_assets\",\n \"find_next_transcribable_campaign_asset\",\n \"find_and_order_potential_transcribable_topic_assets\",\n \"find_new_transcribable_topic_assets\",\n \"find_next_transcribable_topic_asset\",\n \"find_next_transcribable_topic_assets\",\n \"find_transcribable_topic_asset\",\n \"find_invalid_next_transcribable_campaign_assets\",\n \"find_invalid_next_transcribable_topic_assets\",\n]\n"
},
{
"path": "concordia/utils/next_asset/transcribable/campaign.py",
"content": "from typing import Dict\n\nfrom django.db import transaction\nfrom django.db.models import Case, IntegerField, Q, QuerySet, Subquery, When\n\nfrom concordia import models as concordia_models\nfrom concordia.logging import ConcordiaLogger\nfrom concordia.utils.celery import get_registered_task\n\nstructured_logger = ConcordiaLogger.get_logger(__name__)\n\n\ndef _reserved_asset_ids_subq(\n campaign: concordia_models.Campaign,\n) -> \"QuerySet[Dict[str, int]]\":\n \"\"\"\n Return a subquery of reserved asset identifiers for a campaign.\n\n Behavior:\n Produces a subquery suitable for use with `Subquery(...)` and\n `exclude(pk__in=...)` clauses to filter out assets that currently have\n an active reservation.\n\n Args:\n campaign (concordia_models.Campaign): Campaign whose reserved assets\n should be excluded.\n\n Returns:\n QuerySet[Dict[str, int]]: A queryset of dictionaries with a single key\n \"asset_id\" corresponding to reserved assets.\n \"\"\"\n return concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__campaign=campaign\n ).values(\"asset_id\")\n\n\ndef _eligible_transcribable_base_qs(\n campaign: concordia_models.Campaign,\n) -> \"QuerySet[concordia_models.Asset]\":\n \"\"\"\n Build the base queryset of transcribable assets for a campaign.\n\n Behavior:\n Restricts to published projects, items, and assets, and to assets whose\n transcription status is either `NOT_STARTED` or `IN_PROGRESS`.\n\n Args:\n campaign (concordia_models.Campaign): Campaign scope for filtering.\n\n Returns:\n QuerySet[concordia_models.Asset]: Transcribable assets, with `item` and\n `item__project` selected via `select_related`.\n \"\"\"\n return concordia_models.Asset.objects.filter(\n campaign_id=campaign.id,\n item__project__published=True,\n item__published=True,\n published=True,\n transcription_status__in=[\n concordia_models.TranscriptionStatus.NOT_STARTED,\n concordia_models.TranscriptionStatus.IN_PROGRESS,\n ],\n ).select_related(\"item\", \"item__project\")\n\n\ndef _next_seq_after(pk: int | None) -> int | None:\n \"\"\"\n Resolve the sequence number for a given asset primary key.\n\n Behavior:\n Convenience utility for ordering logic when advancing within a series\n of assets.\n\n Args:\n pk (int | None): Asset primary key whose sequence to resolve.\n\n Returns:\n int | None: The asset's sequence number, or None if `pk` is falsy\n or the asset does not exist.\n \"\"\"\n if not pk:\n return None\n return (\n concordia_models.Asset.objects.filter(pk=pk)\n .values_list(\"sequence\", flat=True)\n .first()\n )\n\n\ndef _order_unstarted_first(\n qs: \"QuerySet[concordia_models.Asset]\",\n) -> \"QuerySet[concordia_models.Asset]\":\n \"\"\"\n Apply a stable ordering that prefers `NOT_STARTED` over `IN_PROGRESS`,\n then orders by `sequence`.\n\n Args:\n qs (QuerySet[concordia_models.Asset]): Base queryset to annotate and sort.\n\n Returns:\n QuerySet[concordia_models.Asset]: Annotated and ordered queryset with a\n transient `unstarted` field (1 for `NOT_STARTED`, else 0).\n \"\"\"\n return qs.annotate(\n unstarted=Case(\n When(\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n then=1,\n ),\n default=0,\n output_field=IntegerField(),\n )\n ).order_by(\"-unstarted\", \"sequence\")\n\n\n@transaction.atomic\ndef _find_transcribable_in_item(\n campaign: concordia_models.Campaign,\n *,\n item_id: str,\n after_asset_pk: int | None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Fast path: find the next transcribable asset in the same item.\n\n Behavior:\n - Exclude the current asset.\n - Advance by `(sequence, id)` within the item.\n - Return only `NOT_STARTED` here (defer `IN_PROGRESS` to later fallbacks).\n - Skip reserved assets.\n - Respect published flags on campaign, project, item, and asset.\n\n Args:\n campaign (concordia_models.Campaign): Campaign scope.\n item_id (str): Identifier of the item to stay within.\n after_asset_pk (int | None): Asset primary key to advance from.\n\n Returns:\n concordia_models.Asset | None: The next eligible asset, or None if none.\n \"\"\"\n if not item_id:\n return None\n\n # Find current sequence to advance correctly within the item\n cur_seq = None\n if after_asset_pk:\n cur_seq = (\n concordia_models.Asset.objects.filter(pk=after_asset_pk)\n .values_list(\"sequence\", flat=True)\n .first()\n )\n\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(\n \"asset_id\"\n )\n\n base = concordia_models.Asset.objects.filter(\n item__item_id=item_id,\n item__published=True,\n item__project__published=True,\n published=True,\n campaign_id=campaign.id,\n ).exclude(pk__in=Subquery(reserved_asset_ids))\n\n if after_asset_pk:\n if cur_seq is not None:\n base = base.filter(\n Q(sequence__gt=cur_seq)\n | (Q(sequence=cur_seq) & Q(id__gt=after_asset_pk))\n )\n else:\n base = base.exclude(id=after_asset_pk)\n\n # ONLY NOT_STARTED in this short-circuit\n return (\n base.filter(\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED\n )\n .order_by(\"sequence\", \"id\")\n .first()\n )\n\n\ndef _find_transcribable_not_started_in_project(\n campaign: concordia_models.Campaign,\n *,\n project_slug: str,\n exclude_item_id: str | None = None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Fast path: find the first `NOT_STARTED` asset in the same project.\n\n Behavior:\n Allows different items (optionally excluding the current item to avoid\n bouncing back). Uses a stable ordering by `(item_id, sequence, id)`.\n\n Args:\n campaign (concordia_models.Campaign): Campaign scope.\n project_slug (str): Slug of the project to stay within.\n exclude_item_id (str | None): If provided, exclude this item.\n\n Returns:\n concordia_models.Asset | None: The first eligible asset, or None if none.\n \"\"\"\n if not project_slug:\n return None\n\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(\n \"asset_id\"\n )\n\n base = concordia_models.Asset.objects.filter(\n campaign_id=campaign.id,\n item__project__slug=project_slug,\n item__published=True,\n item__project__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n ).exclude(pk__in=Subquery(reserved_asset_ids))\n\n if exclude_item_id:\n base = base.exclude(item__item_id=exclude_item_id)\n\n return base.order_by(\"item__item_id\", \"sequence\", \"id\").first()\n\n\ndef find_new_transcribable_campaign_assets(\n campaign: concordia_models.Campaign,\n) -> \"QuerySet[concordia_models.Asset]\":\n \"\"\"\n Return assets in a campaign that are eligible to be added to the cache.\n\n Behavior:\n Builds the candidate set for the `NextTranscribableCampaignAsset` cache\n by excluding assets that are not `NOT_STARTED` or `IN_PROGRESS`, assets\n already reserved, and assets already present in the cache.\n\n Args:\n campaign (concordia_models.Campaign): Campaign to filter by.\n\n Returns:\n QuerySet[concordia_models.Asset]: Eligible assets ordered by `sequence`.\n \"\"\"\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__campaign=campaign\n ).values(\"asset_id\")\n next_asset_ids = concordia_models.NextTranscribableCampaignAsset.objects.filter(\n campaign=campaign\n ).values(\"asset_id\")\n\n return (\n concordia_models.Asset.objects.filter(\n campaign_id=campaign.id,\n item__project__published=True,\n item__published=True,\n published=True,\n )\n .filter(\n Q(transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED)\n | Q(transcription_status=concordia_models.TranscriptionStatus.IN_PROGRESS)\n )\n .exclude(pk__in=Subquery(reserved_asset_ids))\n .exclude(pk__in=Subquery(next_asset_ids))\n .order_by(\"sequence\")\n )\n\n\ndef find_next_transcribable_campaign_assets(\n campaign: concordia_models.Campaign,\n) -> \"QuerySet[concordia_models.NextTranscribableCampaignAsset]\":\n \"\"\"\n Return all cached transcribable assets for a campaign.\n\n Behavior:\n Reads from the `NextTranscribableCampaignAsset` cache table for the\n given campaign.\n\n Args:\n campaign (concordia_models.Campaign): Campaign to retrieve cached assets for.\n\n Returns:\n QuerySet[concordia_models.NextTranscribableCampaignAsset]: Cached candidates.\n \"\"\"\n return concordia_models.NextTranscribableCampaignAsset.objects.filter(\n campaign=campaign\n )\n\n\n@transaction.atomic\ndef find_transcribable_campaign_asset(\n campaign: concordia_models.Campaign,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Retrieve a single transcribable asset from the campaign.\n\n Behavior:\n First attempts to select a cached asset from\n `NextTranscribableCampaignAsset`. If none is available, falls back to a\n direct query over `Asset` and triggers a background task to replenish\n the cache.\n\n Concurrency:\n Uses `select_for_update(skip_locked=True, of=(\"self\",))` so only the\n `Asset` row is locked and concurrent consumers skip locked rows.\n\n Args:\n campaign (concordia_models.Campaign): Campaign to search within.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or None if\n unavailable.\n \"\"\"\n next_asset = (\n find_next_transcribable_campaign_assets(campaign)\n .select_for_update(skip_locked=True, of=(\"self\",))\n .values_list(\"asset_id\", flat=True)\n .first()\n )\n\n spawn_task = False\n if next_asset:\n asset_query = concordia_models.Asset.objects.filter(id=next_asset)\n else:\n # No asset in the NextTranscribableCampaignAsset table for this campaign,\n # so fallback to manually finding on\n structured_logger.debug(\n \"No cached assets available, falling back to manual lookup\",\n event_code=\"transcribable_fallback_manual_lookup\",\n campaign=campaign,\n )\n asset_query = find_new_transcribable_campaign_assets(campaign)\n spawn_task = True\n # select_for_update(of=(\"self\",)) causes the row locking only to\n # apply to the Asset table, rather than also locking joined item table\n asset = (\n asset_query.select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n if spawn_task:\n # Spawn a task to populate the table for this campaign\n # We wait to do this until after getting an asset because otherwise there's a\n # a chance all valid assets get grabbed by the task and our query will return\n # nothing\n structured_logger.debug(\n \"Spawned background task to populate cache\",\n event_code=\"transcribable_cache_population_triggered\",\n campaign=campaign,\n )\n populate_task = get_registered_task(\n \"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_campaign\"\n )\n populate_task.delay(campaign.id)\n return asset\n\n\ndef find_and_order_potential_transcribable_campaign_assets(\n campaign: concordia_models.Campaign,\n project_slug: str,\n item_id: str,\n asset_pk: int,\n) -> \"QuerySet[concordia_models.NextTranscribableCampaignAsset]\":\n \"\"\"\n Retrieve and prioritize cached transcribable assets based on proximity\n and status.\n\n Behavior:\n Orders cached candidates from `NextTranscribableCampaignAsset` to prefer:\n - `NOT_STARTED` over `IN_PROGRESS` (via transient `unstarted` flag),\n - same project,\n - same item,\n then by `sequence` and `asset_id` for stability.\n\n Annotations added to each row (transient fields):\n - unstarted (int): 1 if transcription status is `NOT_STARTED`, else 0.\n - same_project (int): 1 if the candidate shares `project_slug`, else 0.\n - same_item (int): 1 if the candidate shares `item_id`, else 0.\n\n Args:\n campaign (concordia_models.Campaign): Campaign to filter by.\n project_slug (str): Slug of the original asset's project.\n item_id (str): Item identifier of the original asset.\n asset_pk (int): Primary key of the original asset.\n\n Returns:\n QuerySet[concordia_models.NextTranscribableCampaignAsset]: Prioritized\n cached candidates.\n \"\"\"\n potential_next_assets = find_next_transcribable_campaign_assets(campaign)\n\n potential_next_assets = potential_next_assets.annotate(\n unstarted=Case(\n When(\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n then=1,\n ),\n default=0,\n output_field=IntegerField(),\n ),\n same_project=Case(\n When(project_slug=project_slug, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n same_item=Case(\n When(item_item_id=item_id, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n ).order_by(\n \"-unstarted\",\n \"-same_project\",\n \"-same_item\",\n \"sequence\",\n \"asset_id\",\n )\n\n return potential_next_assets\n\n\n@transaction.atomic\ndef find_next_transcribable_campaign_asset(\n campaign: concordia_models.Campaign,\n project_slug: str,\n item_id: str,\n original_asset_id: int | None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Retrieve the next best transcribable asset within a campaign.\n\n Priority for short-circuit selection (before cache and fallback):\n 1) If `item_id` is provided, return the next `NOT_STARTED` asset in\n that item by sequence (strictly after the original asset when known).\n 2) If `project_slug` is provided, return the first `NOT_STARTED` asset\n in that project (ordered by item id, then sequence), excluding the\n current item to keep moving forward.\n\n If none of the above match, fall back to the cache-backed path:\n Attempts to retrieve a candidate from `NextTranscribableCampaignAsset`. If\n none is found, compute from `Asset` and trigger cache population.\n\n After exhausting `NOT_STARTED` options, consider `IN_PROGRESS` assets in the\n same item (strictly after the original when known).\n\n Concurrency:\n Uses `select_for_update(skip_locked=True, of=(\"self\",))` to avoid\n double-assignments across concurrent consumers.\n\n Args:\n campaign (concordia_models.Campaign): Campaign to search within.\n project_slug (str): Slug of the current project.\n item_id (str): Identifier of the current item.\n original_asset_id (int | None): Identifier of the asset just transcribed.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or None if\n unavailable.\n \"\"\"\n # Normalize original_asset_id for safe use in filters/comparisons\n try:\n original_pk = int(original_asset_id) if original_asset_id is not None else None\n except (TypeError, ValueError):\n original_pk = None\n\n # Resolve \"after sequence\" only when the original asset belongs to the same item.\n after_seq = None\n if item_id and original_pk is not None:\n try:\n orig = (\n concordia_models.Asset.objects.select_related(\"item\")\n .only(\"id\", \"sequence\", \"item__item_id\")\n .get(pk=original_pk)\n )\n if getattr(orig.item, \"item_id\", None) == item_id:\n after_seq = orig.sequence\n except concordia_models.Asset.DoesNotExist:\n after_seq = None\n\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(\n \"asset_id\"\n )\n\n # Short-circuit: same item and NOT_STARTED after current sequence\n if item_id:\n qs = concordia_models.Asset.objects.filter(\n campaign_id=campaign.id,\n item__item_id=item_id,\n item__published=True,\n item__project__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n ).exclude(pk__in=Subquery(reserved_asset_ids))\n if original_pk is not None:\n qs = qs.exclude(pk=original_pk)\n if after_seq is not None:\n qs = qs.filter(\n Q(sequence__gt=after_seq)\n | (Q(sequence=after_seq) & Q(id__gt=original_pk))\n )\n asset = (\n qs.order_by(\"sequence\", \"id\")\n .select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n if asset:\n return asset\n\n # Short-circuit: same project and NOT_STARTED (avoid current item and original)\n if project_slug:\n candidate = concordia_models.Asset.objects.filter(\n campaign_id=campaign.id,\n item__project__slug=project_slug,\n item__published=True,\n item__project__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n ).exclude(pk__in=Subquery(reserved_asset_ids))\n if original_pk is not None:\n candidate = candidate.exclude(pk=original_pk)\n if item_id:\n candidate = candidate.exclude(item__item_id=item_id)\n\n asset = (\n candidate.order_by(\"item__item_id\", \"sequence\", \"id\")\n .select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n if asset:\n return asset\n\n # Cache-backed selection (NOT_STARTED), then manual fallback (also NOT_STARTED)\n potential_next_assets = find_and_order_potential_transcribable_campaign_assets(\n campaign, project_slug, item_id, original_asset_id\n )\n if original_pk is not None:\n potential_next_assets = potential_next_assets.exclude(asset_id=original_pk)\n if item_id:\n # Keep moving forward: avoid bouncing to the same item\n potential_next_assets = potential_next_assets.exclude(item_item_id=item_id)\n\n asset_id = (\n potential_next_assets.select_for_update(skip_locked=True, of=(\"self\",))\n .values_list(\"asset_id\", flat=True)\n .first()\n )\n\n spawn_task = False\n if asset_id:\n asset_query = concordia_models.Asset.objects.filter(id=asset_id)\n else:\n structured_logger.debug(\n \"No cached assets matched, falling back to manual lookup\",\n event_code=\"transcribable_next_fallback_manual\",\n campaign=campaign,\n )\n spawn_task = True\n asset_query = find_new_transcribable_campaign_assets(campaign)\n if original_pk is not None:\n asset_query = asset_query.exclude(pk=original_pk)\n if item_id:\n asset_query = asset_query.exclude(item__item_id=item_id)\n asset_query = asset_query.annotate(\n unstarted=Case(\n When(\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n then=1,\n ),\n default=0,\n output_field=IntegerField(),\n ),\n same_project=Case(\n When(item__project__slug=project_slug, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n same_item=Case(\n When(item__item_id=item_id, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n ).order_by(\n \"-unstarted\",\n \"-same_project\",\n \"-same_item\",\n \"sequence\",\n \"id\",\n )\n\n asset = (\n asset_query.select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n if asset:\n if spawn_task:\n structured_logger.debug(\n \"Spawned background task to populate cache\",\n event_code=\"transcribable_next_cache_population\",\n campaign=campaign,\n )\n populate_task = get_registered_task(\n \"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_campaign\"\n )\n populate_task.delay(campaign.id)\n return asset\n\n # Only now consider same-item IN_PROGRESS after current sequence\n if item_id:\n qs = concordia_models.Asset.objects.filter(\n campaign_id=campaign.id,\n item__item_id=item_id,\n item__published=True,\n item__project__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.IN_PROGRESS,\n ).exclude(pk__in=Subquery(reserved_asset_ids))\n if original_pk is not None:\n qs = qs.exclude(pk=original_pk)\n if after_seq is not None:\n qs = qs.filter(\n Q(sequence__gt=after_seq)\n | (Q(sequence=after_seq) & Q(id__gt=original_pk))\n )\n asset = (\n qs.order_by(\"sequence\", \"id\")\n .select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n if asset:\n return asset\n\n return None\n\n\ndef find_invalid_next_transcribable_campaign_assets(\n campaign_id: int,\n) -> \"QuerySet[concordia_models.NextTranscribableCampaignAsset]\":\n \"\"\"\n Return cached rows that are invalid for transcription for a campaign.\n\n Behavior:\n Identifies `NextTranscribableCampaignAsset` rows that are no longer valid\n because the underlying asset is neither `NOT_STARTED` nor `IN_PROGRESS`,\n or because the asset is currently reserved.\n\n Args:\n campaign_id (int): Identifier of the campaign.\n\n Returns:\n QuerySet[concordia_models.NextTranscribableCampaignAsset]: Distinct invalid\n cache rows.\n \"\"\"\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__campaign_id=campaign_id\n ).values(\"asset_id\")\n\n # Assets with transcription_status not eligible for transcription\n status_filtered = concordia_models.NextTranscribableCampaignAsset.objects.filter(\n campaign_id=campaign_id\n ).exclude(\n asset__transcription_status__in=[\n concordia_models.TranscriptionStatus.NOT_STARTED,\n concordia_models.TranscriptionStatus.IN_PROGRESS,\n ]\n )\n\n # Assets that are reserved\n reserved_filtered = concordia_models.NextTranscribableCampaignAsset.objects.filter(\n campaign_id=campaign_id, asset_id__in=Subquery(reserved_asset_ids)\n )\n\n return (status_filtered | reserved_filtered).distinct()\n"
},
{
"path": "concordia/utils/next_asset/transcribable/topic.py",
"content": "from typing import Dict\n\nfrom django.db import transaction\nfrom django.db.models import Case, IntegerField, Q, QuerySet, Subquery, When\n\nfrom concordia import models as concordia_models\nfrom concordia.logging import ConcordiaLogger\nfrom concordia.utils.celery import get_registered_task\n\nstructured_logger = ConcordiaLogger.get_logger(__name__)\n\n\ndef _reserved_asset_ids_subq() -> \"QuerySet[Dict[str, int]]\":\n \"\"\"\n Return a subquery of reserved asset identifiers.\n\n Behavior:\n Not filtered to the topic to avoid extra joins. Produces a subquery\n suitable for use with `Subquery(...)` and `exclude(pk__in=...)`\n clauses to filter out assets that currently have an active\n reservation.\n\n Returns:\n QuerySet[Dict[str, int]]: A queryset of dictionaries with a single key\n \"asset_id\" corresponding to reserved assets.\n \"\"\"\n return concordia_models.AssetTranscriptionReservation.objects.values(\"asset_id\")\n\n\ndef _eligible_transcribable_base_qs(\n topic: \"concordia_models.Topic\",\n) -> \"QuerySet[concordia_models.Asset]\":\n \"\"\"\n Build the base queryset of transcribable assets for a topic.\n\n Behavior:\n Restricts to published projects, items, and assets, and to assets whose\n transcription status is either `NOT_STARTED` or `IN_PROGRESS`.\n\n Args:\n topic (concordia_models.Topic): Topic scope for filtering.\n\n Returns:\n QuerySet[concordia_models.Asset]: Transcribable assets, with `item` and\n `item__project` selected via `select_related`.\n \"\"\"\n return concordia_models.Asset.objects.filter(\n item__project__topics=topic.id,\n item__project__published=True,\n item__published=True,\n published=True,\n transcription_status__in=[\n concordia_models.TranscriptionStatus.NOT_STARTED,\n concordia_models.TranscriptionStatus.IN_PROGRESS,\n ],\n ).select_related(\"item\", \"item__project\")\n\n\ndef _next_seq_after(pk: int | None) -> int | None:\n \"\"\"\n Resolve the sequence number for a given asset primary key.\n\n Behavior:\n Convenience utility for ordering logic when advancing within a series\n of assets.\n\n Args:\n pk (int | None): Asset primary key whose sequence to resolve.\n\n Returns:\n int | None: The asset's sequence number, or None if `pk` is falsy\n or the asset does not exist.\n \"\"\"\n if not pk:\n return None\n return (\n concordia_models.Asset.objects.filter(pk=pk)\n .values_list(\"sequence\", flat=True)\n .first()\n )\n\n\ndef _order_unstarted_first(\n qs: \"QuerySet[concordia_models.Asset]\",\n) -> \"QuerySet[concordia_models.Asset]\":\n \"\"\"\n Apply a stable ordering that prefers `NOT_STARTED` over `IN_PROGRESS`,\n then orders by `sequence`.\n\n Args:\n qs (QuerySet[concordia_models.Asset]): Base queryset to annotate and sort.\n\n Returns:\n QuerySet[concordia_models.Asset]: Annotated and ordered queryset with a\n transient `unstarted` field (1 for `NOT_STARTED`, else 0).\n \"\"\"\n return qs.annotate(\n unstarted=Case(\n When(\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n then=1,\n ),\n default=0,\n output_field=IntegerField(),\n )\n ).order_by(\"-unstarted\", \"sequence\")\n\n\ndef _find_transcribable_in_item_for_topic(\n topic: \"concordia_models.Topic\",\n *,\n item_id: str,\n after_asset_pk: int | None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Fast path: find the next transcribable asset in the same item, constrained\n to the topic.\n\n Behavior:\n - Asset must belong to a project that is in this topic.\n - Exclude the current asset.\n - Advance by `(sequence, id)` within the item.\n - Return only `NOT_STARTED` here (defer `IN_PROGRESS` to later fallbacks).\n - Skip reserved assets.\n - Respect published flags.\n\n Args:\n topic (concordia_models.Topic): Topic scope.\n item_id (str): Identifier of the item to stay within.\n after_asset_pk (int | None): Asset primary key to advance from.\n\n Returns:\n concordia_models.Asset | None: The next eligible asset, or None if none.\n \"\"\"\n if not item_id:\n return None\n\n cur_seq = None\n if after_asset_pk:\n cur_seq = (\n concordia_models.Asset.objects.filter(pk=after_asset_pk)\n .values_list(\"sequence\", flat=True)\n .first()\n )\n\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(\n \"asset_id\"\n )\n\n base = concordia_models.Asset.objects.filter(\n item__item_id=item_id,\n item__project__topics=topic.id,\n item__published=True,\n item__project__published=True,\n published=True,\n ).exclude(pk__in=Subquery(reserved_asset_ids))\n\n if after_asset_pk:\n if cur_seq is not None:\n base = base.filter(\n Q(sequence__gt=cur_seq)\n | (Q(sequence=cur_seq) & Q(id__gt=after_asset_pk))\n )\n else:\n base = base.exclude(id=after_asset_pk)\n\n # ONLY NOT_STARTED in this short-circuit\n return (\n base.filter(\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED\n )\n .order_by(\"sequence\", \"id\")\n .first()\n )\n\n\ndef _find_transcribable_not_started_in_project_for_topic(\n topic: \"concordia_models.Topic\",\n *,\n project_slug: str,\n exclude_item_id: str | None = None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Fast path: find the first `NOT_STARTED` asset in the same project within\n this topic.\n\n Behavior:\n Optionally exclude the current item. Uses a stable ordering by\n `(item__item_id, sequence, id)`.\n\n Args:\n topic (concordia_models.Topic): Topic scope.\n project_slug (str): Slug of the project to stay within.\n exclude_item_id (str | None): If provided, exclude this item.\n\n Returns:\n concordia_models.Asset | None: The first eligible asset, or None if none.\n \"\"\"\n if not project_slug:\n return None\n\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(\n \"asset_id\"\n )\n\n base = concordia_models.Asset.objects.filter(\n item__project__topics=topic.id,\n item__project__slug=project_slug,\n item__published=True,\n item__project__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n ).exclude(pk__in=Subquery(reserved_asset_ids))\n\n if exclude_item_id:\n base = base.exclude(item__item_id=exclude_item_id)\n\n return base.order_by(\"item__item_id\", \"sequence\", \"id\").first()\n\n\ndef find_new_transcribable_topic_assets(\n topic: \"concordia_models.Topic\",\n) -> \"QuerySet[concordia_models.Asset]\":\n \"\"\"\n Return assets in a topic that are eligible to be added to the cache.\n\n Behavior:\n Builds the candidate set for the `NextTranscribableTopicAsset` cache by\n excluding assets that are not `NOT_STARTED` or `IN_PROGRESS`, assets\n already reserved, and assets already present in the cache.\n\n Args:\n topic (concordia_models.Topic): Topic to filter by.\n\n Returns:\n QuerySet[concordia_models.Asset]: Eligible assets ordered by `sequence`.\n \"\"\"\n # Filtering this to the topic would be more costly than just getting all ids\n # in most cases because it requires joining the asset table to the item table to\n # the project table to the topic table.\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(\n \"asset_id\"\n )\n next_asset_ids = concordia_models.NextTranscribableTopicAsset.objects.filter(\n topic=topic\n ).values(\"asset_id\")\n\n return (\n concordia_models.Asset.objects.filter(\n item__project__topics=topic.id,\n item__project__published=True,\n item__published=True,\n published=True,\n )\n .filter(\n Q(transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED)\n | Q(transcription_status=concordia_models.TranscriptionStatus.IN_PROGRESS)\n )\n .exclude(pk__in=Subquery(reserved_asset_ids))\n .exclude(pk__in=Subquery(next_asset_ids))\n .order_by(\"sequence\")\n )\n\n\ndef find_next_transcribable_topic_assets(\n topic: \"concordia_models.Topic\",\n) -> \"QuerySet[concordia_models.NextTranscribableTopicAsset]\":\n \"\"\"\n Return all cached transcribable assets for a topic.\n\n Behavior:\n Reads from the `NextTranscribableTopicAsset` cache table for the\n given topic.\n\n Args:\n topic (concordia_models.Topic): Topic to retrieve cached assets for.\n\n Returns:\n QuerySet[concordia_models.NextTranscribableTopicAsset]: Cached candidates.\n \"\"\"\n return concordia_models.NextTranscribableTopicAsset.objects.filter(topic=topic)\n\n\n@transaction.atomic\ndef find_transcribable_topic_asset(\n topic: \"concordia_models.Topic\",\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Retrieve a single transcribable asset from the topic.\n\n Behavior:\n First attempts to select a cached asset from\n `NextTranscribableTopicAsset`. If none is available, falls back to a\n direct query over `Asset` and triggers a background task to replenish\n the cache.\n\n Concurrency:\n Uses `select_for_update(skip_locked=True, of=(\"self\",))` so only the\n `Asset` row is locked and concurrent consumers skip locked rows.\n\n Args:\n topic (concordia_models.Topic): Topic to search within.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or None if\n unavailable.\n \"\"\"\n next_asset = (\n find_next_transcribable_topic_assets(topic)\n .select_for_update(skip_locked=True, of=(\"self\",))\n .values_list(\"asset_id\", flat=True)\n .first()\n )\n\n spawn_task = False\n if next_asset:\n asset_query = concordia_models.Asset.objects.filter(id=next_asset)\n else:\n # No asset in the NextTranscribableTopicAsset table for this topic,\n # so fallback to manually finding one\n structured_logger.debug(\n \"No cached assets available, falling back to manual lookup\",\n event_code=\"transcribable_fallback_manual_lookup\",\n topic=topic,\n )\n asset_query = find_new_transcribable_topic_assets(topic)\n spawn_task = True\n # select_for_update(of=(\"self\",)) causes the row locking only to\n # apply to the Asset table, rather than also locking joined item table\n asset = (\n asset_query.select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n if spawn_task:\n # Spawn a task to populate the table for this topic\n # We wait to do this until after getting an asset because otherwise there's a\n # a chance all valid assets get grabbed by the task and our query will return\n # nothing\n structured_logger.debug(\n \"Spawned background task to populate cache\",\n event_code=\"transcribable_cache_population_triggered\",\n topic=topic,\n )\n populate_task = get_registered_task(\n \"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_topic\"\n )\n populate_task.delay(topic.id)\n return asset\n\n\ndef find_and_order_potential_transcribable_topic_assets(\n topic: \"concordia_models.Topic\",\n project_slug: str,\n item_id: str,\n asset_pk: int,\n) -> \"QuerySet[concordia_models.NextTranscribableTopicAsset]\":\n \"\"\"\n Retrieve and prioritize cached transcribable assets based on proximity\n and status.\n\n Behavior:\n Orders cached candidates from `NextTranscribableTopicAsset` to prefer:\n - `NOT_STARTED` over `IN_PROGRESS` (via transient `unstarted` flag),\n - same project,\n - same item,\n then by `sequence` and `asset_id` for stability.\n\n Annotations added to each row (transient fields):\n - unstarted (int): 1 if transcription status is `NOT_STARTED`, else 0.\n - same_project (int): 1 if the candidate shares `project_slug`, else 0.\n - same_item (int): 1 if the candidate shares `item_id`, else 0.\n\n Args:\n topic (concordia_models.Topic): Topic to filter by.\n project_slug (str): Slug of the original asset's project.\n item_id (str): Item identifier of the original asset.\n asset_pk (int): Primary key of the original asset.\n\n Returns:\n QuerySet[concordia_models.NextTranscribableTopicAsset]: Prioritized\n cached candidates.\n \"\"\"\n potential_next_assets = find_next_transcribable_topic_assets(topic)\n\n potential_next_assets = potential_next_assets.annotate(\n unstarted=Case(\n When(\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n then=1,\n ),\n default=0,\n output_field=IntegerField(),\n ),\n same_project=Case(\n When(project_slug=project_slug, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n same_item=Case(\n When(item_item_id=item_id, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n ).order_by(\n \"-unstarted\",\n \"-same_project\",\n \"-same_item\",\n \"sequence\",\n \"asset_id\",\n )\n\n return potential_next_assets\n\n\n@transaction.atomic\ndef find_next_transcribable_topic_asset(\n topic: \"concordia_models.Topic\",\n project_slug: str,\n item_id: str,\n original_asset_id: int | None,\n) -> \"concordia_models.Asset | None\":\n \"\"\"\n Retrieve the next best transcribable asset within a topic.\n\n Priority for short-circuit selection (before cache and fallback):\n 1) If `item_id` is provided, return the next `NOT_STARTED` asset in\n that item by sequence (strictly after the original asset when known).\n 2) If `project_slug` is provided, return the first `NOT_STARTED` asset\n in that project (ordered by item id, then sequence), excluding the\n current item to keep moving forward.\n\n If none of the above match, fall back to the cache-backed path:\n Attempts to retrieve a candidate from `NextTranscribableTopicAsset`. If\n none is found, compute from `Asset` and trigger cache population.\n\n After exhausting `NOT_STARTED` options, consider `IN_PROGRESS` assets in the\n same item (strictly after the original when known).\n\n Concurrency:\n Uses `select_for_update(skip_locked=True, of=(\"self\",))` to avoid\n double-assignments across concurrent consumers.\n\n Args:\n topic (concordia_models.Topic): Topic to search within.\n project_slug (str): Slug of the current project.\n item_id (str): Identifier of the current item.\n original_asset_id (int | None): Identifier of the asset just transcribed.\n\n Returns:\n concordia_models.Asset | None: A locked eligible asset, or None if\n unavailable.\n \"\"\"\n # Resolve original context safely (int or digit-string only)\n after_seq = None\n orig = None\n orig_item_id = None\n orig_id_valid = isinstance(original_asset_id, int) or (\n isinstance(original_asset_id, str) and original_asset_id.isdigit()\n )\n if orig_id_valid:\n try:\n orig = (\n concordia_models.Asset.objects.select_related(\"item\")\n .only(\"id\", \"sequence\", \"item__item_id\")\n .get(pk=original_asset_id)\n )\n orig_item_id = getattr(orig.item, \"item_id\", None)\n # Keep sequence handy for same-item gating in any path\n after_seq = orig.sequence\n except concordia_models.Asset.DoesNotExist:\n orig = None\n orig_item_id = None\n after_seq = None\n\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(\n \"asset_id\"\n )\n\n # Short-circuit: same item and NOT_STARTED after current sequence\n if item_id:\n qs = concordia_models.Asset.objects.filter(\n item__project__topics=topic.id,\n item__item_id=item_id,\n item__published=True,\n item__project__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n ).exclude(pk__in=Subquery(reserved_asset_ids))\n if orig_id_valid:\n qs = qs.exclude(pk=original_asset_id)\n if after_seq is not None and orig_item_id == item_id:\n qs = qs.filter(\n Q(sequence__gt=after_seq)\n | (Q(sequence=after_seq) & Q(id__gt=int(original_asset_id)))\n )\n asset = (\n qs.order_by(\"sequence\", \"id\")\n .select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n if asset:\n return asset\n\n # Short-circuit: same project and NOT_STARTED (topic-constrained)\n if project_slug:\n candidate = concordia_models.Asset.objects.filter(\n item__project__topics=topic.id,\n item__project__slug=project_slug,\n item__published=True,\n item__project__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n ).exclude(pk__in=Subquery(reserved_asset_ids))\n if orig_id_valid:\n candidate = candidate.exclude(pk=original_asset_id)\n if item_id:\n candidate = candidate.exclude(item__item_id=item_id)\n\n asset = (\n candidate.order_by(\"item__item_id\", \"sequence\", \"id\")\n .select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n if asset:\n return asset\n\n # Cache-backed selection (NOT_STARTED anywhere), then manual fallback.\n potential_next_assets = find_and_order_potential_transcribable_topic_assets(\n topic, project_slug, item_id, original_asset_id\n )\n if orig_id_valid:\n potential_next_assets = potential_next_assets.exclude(\n asset_id=original_asset_id\n )\n if item_id:\n potential_next_assets = potential_next_assets.exclude(item_item_id=item_id)\n\n asset_id = (\n potential_next_assets.select_for_update(skip_locked=True, of=(\"self\",))\n .values_list(\"asset_id\", flat=True)\n .first()\n )\n\n spawn_task = False\n if asset_id:\n asset_query = concordia_models.Asset.objects.filter(id=asset_id)\n else:\n structured_logger.debug(\n \"No cached assets matched, falling back to manual lookup\",\n event_code=\"transcribable_next_fallback_manual\",\n topic=topic,\n )\n spawn_task = True\n asset_query = find_new_transcribable_topic_assets(topic)\n if orig_id_valid:\n asset_query = asset_query.exclude(pk=original_asset_id)\n if item_id:\n asset_query = asset_query.exclude(item__item_id=item_id)\n # If we know the original's item/seq, keep moving forward within that item\n if orig_item_id and after_seq is not None:\n asset_query = asset_query.exclude(\n Q(item__item_id=orig_item_id, sequence__lte=after_seq)\n )\n\n # Prefer same project and same item; if item_id is blank, prefer original's item\n ref_item_id = item_id or orig_item_id\n asset_query = asset_query.annotate(\n unstarted=Case(\n When(\n transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,\n then=1,\n ),\n default=0,\n output_field=IntegerField(),\n ),\n same_project=Case(\n When(item__project__slug=project_slug, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n same_item=Case(\n When(item__item_id=ref_item_id, then=1),\n default=0,\n output_field=IntegerField(),\n ),\n ).order_by(\n \"-unstarted\",\n \"-same_project\",\n \"-same_item\",\n \"sequence\",\n \"id\",\n )\n\n asset = (\n asset_query.select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n if asset:\n if spawn_task:\n structured_logger.debug(\n \"Spawned background task to populate cache\",\n event_code=\"transcribable_next_cache_population\",\n topic=topic,\n )\n populate_task = get_registered_task(\n \"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_topic\"\n )\n populate_task.delay(topic.id)\n return asset\n\n # Only now consider same-item IN_PROGRESS after current sequence\n if item_id:\n qs = concordia_models.Asset.objects.filter(\n item__project__topics=topic.id,\n item__item_id=item_id,\n item__published=True,\n item__project__published=True,\n published=True,\n transcription_status=concordia_models.TranscriptionStatus.IN_PROGRESS,\n ).exclude(pk__in=Subquery(reserved_asset_ids))\n if orig_id_valid:\n qs = qs.exclude(pk=original_asset_id)\n if after_seq is not None and orig_item_id == item_id:\n qs = qs.filter(\n Q(sequence__gt=after_seq)\n | (Q(sequence=after_seq) & Q(id__gt=int(original_asset_id)))\n )\n asset = (\n qs.order_by(\"sequence\", \"id\")\n .select_for_update(skip_locked=True, of=(\"self\",))\n .select_related(\"item\", \"item__project\")\n .first()\n )\n if asset:\n if spawn_task:\n structured_logger.debug(\n \"Spawned background task to populate cache\",\n event_code=\"transcribable_next_cache_population\",\n topic=topic,\n )\n populate_task = get_registered_task(\n \"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_topic\"\n )\n populate_task.delay(topic.id)\n return asset\n\n return None\n\n\ndef find_invalid_next_transcribable_topic_assets(\n topic_id: int,\n) -> \"QuerySet[concordia_models.NextTranscribableTopicAsset]\":\n \"\"\"\n Return cached rows that are invalid for transcription for a topic.\n\n Behavior:\n Identifies `NextTranscribableTopicAsset` rows that are no longer valid\n because the underlying asset is neither `NOT_STARTED` nor\n `IN_PROGRESS`, or because the asset is currently reserved.\n\n Args:\n topic_id (int): Identifier of the topic.\n\n Returns:\n QuerySet[concordia_models.NextTranscribableTopicAsset]: Distinct invalid\n cache rows.\n \"\"\"\n reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(\n asset__item__project__topics=topic_id\n ).values(\"asset_id\")\n\n status_filtered = concordia_models.NextTranscribableTopicAsset.objects.filter(\n topic_id=topic_id\n ).exclude(\n asset__transcription_status__in=[\n concordia_models.TranscriptionStatus.NOT_STARTED,\n concordia_models.TranscriptionStatus.IN_PROGRESS,\n ]\n )\n\n reserved_filtered = concordia_models.NextTranscribableTopicAsset.objects.filter(\n topic_id=topic_id, asset_id__in=Subquery(reserved_asset_ids)\n )\n\n return (status_filtered | reserved_filtered).distinct()\n"
},
{
"path": "concordia/validators.py",
"content": "from django.conf import settings\nfrom django.utils.translation import gettext_lazy as _\n\nfrom .passwords.validators import ComplexityValidator\n\n\nclass DjangoPasswordsValidator(object):\n \"\"\"\n Wrapper for the django-passwords complexity validator which is compatible\n with the Django 1.9+ password validation API\n Because django-passwords is not compatible with Django 4+, the validator\n has been integrated into the concordia.passwords module instead.\n \"\"\"\n\n message = _(\"Must be more complex (%s)\")\n code = \"complexity\"\n\n def __init__(self):\n self.validator = ComplexityValidator(settings.PASSWORD_COMPLEXITY)\n\n def get_help_text(self):\n return _(\"Your password fails to meet our complexity requirements.\")\n\n def validate(self, value, user=None):\n return self.validator(value)\n"
},
{
"path": "concordia/version.py",
"content": "import functools\n\nfrom setuptools_scm import get_version\n\n\n@functools.lru_cache(maxsize=None)\ndef get_concordia_version():\n return get_version()\n"
},
{
"path": "concordia/views/README.md",
"content": "## **init**.py\n\nInitializes the `views` module and re-exports submodules for chained attribute access\nsuch as `views.campaigns`. Also includes some basic views.\n\n### Class-based Views\n\n- **HomeView** - A `ListView` displaying featured campaigns on the homepage\n\n### Function-based Views\n\n- **healthz** - Returns a JSON response with system and application status\n\n## accounts.py\n\nViews related to user account management, including login, registration and profile updates\n\n### Class-based Views\n\n- **ConcordiaPasswordResetConfirmView** - Customized password reset confirmation\n- **ConcordiaPasswordResetRequestView** - Customized password reset requests\n- **ConcordiaRegistrationView** - Custom registration view with rate limiting\n- **ConcordiaLoginView** - Login view with Turnstile challenge validation\n- **AccountProfileView** - View for managing a user's profile and displaying contributions\n- **AccountDeletionView** - View for users to delete their account (or anonymize it)\n- **EmailReconfirmationView** - Handles confirming a user's changed email address\n\n### Function-based Views\n\n- **account_letter** - Generates and returns a PDF letter summarizing contributions\n- **get_pages** - Renders a fragment of recent contributed pages\n\n### Functions\n\n- **registration_rate** - Rate-limit for failed registration attempts\n\n## ajax.py\n\nAJAX endpoints for dynamic client interactions\n\n### Function-based Views\n\n- **ajax_session_status** - Returns user-specific session data used by the frontend\n- **ajax_messages** - Returns any queued messages for the current user\n- **generate_ocr_transcription** - Generates a new transcription using OCR\n- **rollback_transcription** - Reverts transcription to the previous version\n- **rollforward_transcription** - Reverts the most recent transcription rollback\n- **save_transcription** - Saves a new transcription\n- **submit_transcription** - Marks a transcription as submitted for review\n- **review_transcription** - Accepts or rejects a submitted transcription\n- **submit_tags** - Updates the tag list for an asset\n- **reserve_asset** - Manages reservation of an asset to prevent conflicts\n\n### Functions\n\n- **get_transcription_superseded** - Determines if the superseded transcription is valid\n- **update_reservation** - Updates the timestamp for a reservation\n- **obtain_reservation** - Creates a new reservation\n\n## assets.py\n\nViews for displaying asset detail pages and redirecting users to the next appropriate asset\n\n### Class-based Views\n\n- **AssetDetailView** - Displays the transcription interface for a single asset\n\n### Function-based Views\n\n- **redirect_to_next_asset** - Redirects to provided asset\n- **redirect_to_next_reviewable_asset** - Finds and redirects to a reviewable asset\n- **redirect_to_next_transcribable_asset** - Finds and redirects to a transcribable asset\n- **redirect_to_next_reviewable_campaign_asset** - Finds and redirects to the a reviewable asset for a campaign\n- **redirect_to_next_transcribable_campaign_asset** - Finds and redirects to a transcribable asset for a campaign\n- **redirect_to_next_reviewable_topic_asset** - Finds and redirects to a reviewable asset for a topic\n- **redirect_to_next_transcribable_topic_asset** - Finds and redirects to a transcribable asset for a topic\n\n## campaigns.py\n\nViews for listing campaigns, rendering campaign details, showing reports and filtering by reviewable status\n\n### Class-based Views\n\n- **CampaignListView** - Lists all active campaigns (unused)\n- **CompletedCampaignListView** - Lists all completed and retired campaigns\n- **CampaignTopicListView** - Primary active campaign list view; also includes active topics\n- **CampaignDetailView** - Shows full details about a single campaign\n- **FilteredCampaignDetailView** - Variant of `CampaignDetailView` that applies filtering based on the user\n- **ReportCampaignView** - Displays a campaign report summarizing stats such as asset counts and contributors\n\n## decorators.py\n\nCustom decorators used by views\n\n### Functions\n\n- **default_cache_control** - Applies default public caching headers for pages that don't vary per user\n- **user_cache_control** - Applies public caching headers with variation for logged-in users\n- **validate_anonymous_user** - Validates anonymous users via Turnstile before processing requests\n- **reserve_rate** - Returns a rate-limit value for unauthenticated users for reserving assets\n- **next_asset_rate** - Returns a rate-limit value for unauthenicated users for next\\_\\*\\_asset views\n\n## items.py\n\nViews for displaying individual item detail pages\n\n### Class-based Views\n\n- **ItemDetailView** - Displays a paginated list of assets within an item\n- **FilteredItemDetailView** - Variant of `ItemDetailView` that applies filtering based on the user\n\n## maintenance_mode.py\n\nViews for toggling the site's maintenance mode. Only accessible to superusers\n\n### Function-based Views\n\n- **maintenance_mode_off** - Disables maintenance mode\n- **maintenance_mode_on** - Enables maintenance mode\n- **maintenance_mode_frontend_available** - Enables access to the frontend for staff while in maintenance mode\n- **maintenance_mode_frontend_unavailable** - Disables access to the frontend for staff while in maintenance mode\n\n## projects.py\n\nViews for displaying project detail pages\n\n### Class-based Views\n\n- **ProjectDetailView** - Displays a project and its items\n- **FilteredProjectDetailView** - Variant of `ProjectDetailView` that applies filtering based on the user\n\n## rate_limit.py\n\nCustom handler for responding to requests that exceed rate limits\n\n### Function-based Views\n\n- **ratelimit_view** - Returns a 429 response when a user is rate-limited\n\n## simple_pages.py\n\nViews and redirects for rendering static pages stored in the database\n\n### Function-based Views\n\n- **simple_page** - Renders a simple static page from the database\n- **about_simple_page** - Renders the \"about\" simple page, which includes some additional data\n\n### Class-based Views\n\n- **HelpCenterRedirectView** - Redirects old help center URLs to new equivalents\n- **HelpCenterSpanishRedirectView** - Redirects old Spanish help center URLs to new equivalents\n\n## topics.py\n\nView for displaying a topic's detail page\n\n### Class-based Views\n\n- **TopicDetailView** - Displays a topic's associated projects\n\n## utils.py\n\nUtility functions, constants and mixins used throughout the views module\n\n### Constants\n\n- **ASSETS_PER_PAGE** - Default number of assets to show per page\n- **PROJECTS_PER_PAGE** - Default number of projects to show per page\n- **ITEMS_PER_PAGE** - Default number of items to show per page\n- **URL_REGEX** - Regular expression used to detect URLs in transcription text\n- **MESSAGE_LEVEL_NAMES** - Dictionary mapping Django message levels to lowercase names\n\n### Functions\n\n- **\\_get_pages** - Returns a queryset of assets a user has worked on\n- **calculate_asset_stats** - Adds contributor and transcription status to the provided assets\n- **annotate_children_with_progress_stats** - Annotates a list of objects with progress information\n\n### Classes\n\n- **AnonymousUserValidationCheckMixin** - Requires anonymous users to pass Turnstile validation\n\n## visualization.py\n\nViews for displaying visualizations\n\n### Classes\n\n- **VisualizationDataView** - Returns JSON representing the visualization `name`\n"
},
{
"path": "concordia/views/__init__.py",
"content": "import json\nimport logging\nimport os\nfrom time import time\n\nfrom django.conf import settings\nfrom django.http import HttpResponse\nfrom django.utils.decorators import method_decorator\nfrom django.views.decorators.cache import never_cache\nfrom django.views.generic import ListView\n\nfrom concordia.models import Banner, Campaign, CarouselSlide\nfrom concordia.version import get_concordia_version\n\n# These imports are required to make chainted attribute access like, e.g.,\n# views.campaigns.CampaignDetailView work correctly\nfrom . import (\n accounts, # noqa: F401\n ajax, # noqa: F401\n assets, # noqa: F401\n campaigns, # noqa: F401\n items, # noqa: F401\n maintenance_mode, # noqa: F401\n projects, # noqa: F401\n rate_limit, # noqa: F401\n simple_pages, # noqa: F401\n topics, # noqa: F401\n visualizations, # noqa: F401\n)\nfrom .decorators import default_cache_control\n\nlogger = logging.getLogger(__name__)\n\n\n@never_cache\ndef healthz(request):\n status = {\n \"current_time\": time(),\n \"load_average\": os.getloadavg(),\n \"debug\": settings.DEBUG,\n }\n\n # We don't want to query a large table but we do want to hit the database\n # at last once:\n status[\"database_has_data\"] = Campaign.objects.count() > 0\n\n status[\"application_version\"] = get_concordia_version()\n\n return HttpResponse(content=json.dumps(status), content_type=\"application/json\")\n\n\n@method_decorator(default_cache_control, name=\"dispatch\")\nclass HomeView(ListView):\n template_name = \"home.html\"\n\n queryset = (\n Campaign.objects.published()\n .listed()\n .filter(display_on_homepage=True)\n .order_by(\"ordering\", \"title\")\n )\n context_object_name = \"campaigns\"\n\n def get_context_data(self, *args, **kwargs):\n ctx = super().get_context_data(*args, **kwargs)\n\n banner = Banner.objects.filter(active=True).first()\n\n if banner is not None:\n ctx[\"banner\"] = banner\n\n ctx[\"slides\"] = CarouselSlide.objects.published().order_by(\"ordering\")\n\n if ctx[\"slides\"]:\n ctx[\"firstslide\"] = ctx[\"slides\"][0]\n\n return ctx\n"
},
{
"path": "concordia/views/accounts.py",
"content": "import gc\nimport logging\nimport tracemalloc\nimport uuid\nfrom smtplib import SMTPException\nfrom typing import Any, Optional, Type\n\nfrom django.conf import settings\nfrom django.contrib import messages\nfrom django.contrib.auth import logout\nfrom django.contrib.auth.decorators import login_required\nfrom django.contrib.auth.mixins import LoginRequiredMixin\nfrom django.contrib.auth.views import (\n LoginView,\n PasswordResetConfirmView,\n PasswordResetView,\n)\nfrom django.contrib.sites.shortcuts import get_current_site\nfrom django.core import signing\nfrom django.core.exceptions import ValidationError\nfrom django.core.mail import send_mail\nfrom django.core.paginator import Paginator\nfrom django.db.models import Sum\nfrom django.forms import Form\nfrom django.http import HttpRequest, HttpResponse, HttpResponseRedirect, JsonResponse\nfrom django.shortcuts import redirect\nfrom django.template import loader\nfrom django.template.loader import render_to_string\nfrom django.urls import reverse_lazy\nfrom django.utils.decorators import method_decorator\nfrom django.utils.translation import gettext_lazy as _\nfrom django.views.decorators.cache import never_cache\nfrom django.views.generic import FormView, ListView, TemplateView\nfrom django_ratelimit.decorators import ratelimit\nfrom django_registration.backends.activation.views import RegistrationView\nfrom weasyprint import HTML\n\nfrom concordia.forms import (\n AccountDeletionForm,\n ActivateAndSetPasswordForm,\n AllowInactivePasswordResetForm,\n TurnstileForm,\n UserLoginForm,\n UserNameForm,\n UserProfileForm,\n UserRegistrationForm,\n)\nfrom concordia.logging import ConcordiaLogger\nfrom concordia.models import Campaign, ConcordiaUser, UserProfileActivity\n\nfrom .utils import _get_pages\n\nlogger = logging.getLogger(__name__)\nstructured_logger = ConcordiaLogger.get_logger(__name__)\n\n\nclass ConcordiaPasswordResetConfirmView(PasswordResetConfirmView):\n \"\"\"\n Confirm a password reset and automatically log in the user.\n\n Extends Djangoβs built-in\n [PasswordResetConfirmView](https://docs.djangoproject.com/en/stable/topics/auth/default/#django.contrib.auth.views.PasswordResetConfirmView)\n to use a custom form and enable automatic login after a successful reset.\n\n Attributes:\n post_reset_login (bool): Whether to log the user in after resetting\n the password.\n form_class (Form): The form used to set the new password and activate\n the account.\n\n Returns:\n response (HttpResponse): Renders the password reset confirmation page or\n redirects after successful password change and login.\n \"\"\"\n\n post_reset_login: bool = True\n form_class: type[Form] = ActivateAndSetPasswordForm\n\n\nclass ConcordiaPasswordResetRequestView(PasswordResetView):\n \"\"\"\n Request a password reset, supporting inactive users.\n\n Extends Djangoβs built-in\n [`PasswordResetView`](https://docs.djangoproject.com/en/stable/topics/auth/default/#django.contrib.auth.views.PasswordResetView)\n to use a custom form that allows inactive users to reset their password\n and activate their account in one step.\n\n Attributes:\n form_class (Form): The form used to validate and process the password\n reset request.\n\n Returns:\n response (HttpResponse): Renders the password reset form or redirects\n after successful submission.\n \"\"\"\n\n form_class: type[Form] = AllowInactivePasswordResetForm\n\n\ndef registration_rate(group: str, request: HttpRequest) -> Optional[str]:\n \"\"\"\n Determine the throttling rate for registration attempts.\n\n Used with the\n [ratelimit](https://django-ratelimit.readthedocs.io/en/stable/usage.html#ratelimit)\n decorator from `django-ratelimit` to dynamically adjust the request rate based\n on form validation.\n\n If the submitted form is invalid, limits requests to 10 per hour. If the\n form is valid, allows the request without throttling.\n\n Args:\n group (str): The rate limit group name. Example: `\"registration\"`\n request (HttpRequest): The request containing registration form data.\n\n Returns:\n rate (str or None): The rate limit string (e.g., \"10/h\") if the form is\n invalid; otherwise `None` to indicate no throttling.\n \"\"\"\n registration_form = UserRegistrationForm(request.POST)\n user = getattr(request, \"user\", None)\n if registration_form.is_valid():\n structured_logger.debug(\n \"Registration form valid.\",\n event_code=\"registration_rate_ok\",\n user=user,\n )\n return None\n else:\n structured_logger.debug(\n \"Registration form invalid, throttling.\",\n event_code=\"registration_rate_throttle\",\n user=user,\n )\n return \"10/h\"\n\n\n@method_decorator(never_cache, name=\"dispatch\")\n@method_decorator(\n ratelimit(\n group=\"registration\",\n key=\"header:cf-connecting-ip\",\n rate=registration_rate,\n method=\"POST\",\n block=settings.RATELIMIT_BLOCK,\n ),\n name=\"post\",\n)\nclass ConcordiaRegistrationView(RegistrationView):\n \"\"\"\n User registration view with rate limiting.\n\n Extends\n [django_registration.views.RegistrationView](https://django-registration.readthedocs.io/en/stable/views.html#django_registration.views.RegistrationView)\n to apply a POST-specific rate limit using the\n [django-ratelimit](https://django-ratelimit.readthedocs.io/en/stable/usage.html#ratelimit)\n decorator. This protects against abuse by restricting failed registration attempts\n while allowing valid submissions to proceed freely.\n\n Attributes:\n form_class (Form): The form used to collect and validate user registration\n data. Example: `UserRegistrationForm`.\n\n Returns:\n response (HttpResponse): Renders the registration form or redirects after\n successful registration.\n \"\"\"\n\n form_class: Type[Form] = UserRegistrationForm\n\n\n@method_decorator(never_cache, name=\"dispatch\")\nclass ConcordiaLoginView(LoginView):\n \"\"\"\n Login view with Turnstile validation.\n\n Extends Django's\n [LoginView](https://docs.djangoproject.com/en/stable/topics/auth/default/#django.contrib.auth.views.LoginView)\n to integrate Turnstile validation during POST requests.\n\n Attributes:\n form_class (Form): The login form used to authenticate users.\n\n Returns:\n response (HttpResponse): The rendered login form or redirect response,\n depending on the validation outcome.\n\n Return Behavior:\n - On GET: Renders the login form with the embedded Turnstile widget.\n - On POST:\n - If both login and Turnstile succeed: redirects to the next page.\n - If Turnstile fails: returns the login form with an error message.\n - If the login form is invalid: returns the form with validation errors.\n \"\"\"\n\n form_class = UserLoginForm\n\n def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:\n structured_logger.debug(\n \"Login POST received.\",\n event_code=\"login_post_entry\",\n user=request.user,\n )\n form = self.get_form()\n if form.is_valid():\n turnstile_form = TurnstileForm(request.POST)\n if turnstile_form.is_valid():\n structured_logger.debug(\n \"Login and Turnstile ok.\",\n event_code=\"login_success\",\n user=request.user,\n )\n return self.form_valid(form)\n else:\n structured_logger.warning(\n \"Turnstile failed for login.\",\n event_code=\"login_turnstile_failed\",\n reason=\"Turnstile validation failed\",\n reason_code=\"turnstile_failed\",\n user=request.user,\n )\n form.add_error(\n None, \"Unable to validate. Please login or complete the challenge.\"\n )\n return self.form_invalid(form)\n\n else:\n structured_logger.debug(\n \"Login form invalid.\",\n event_code=\"login_form_invalid\",\n user=request.user,\n )\n return self.form_invalid(form)\n\n def get_context_data(self, **kwargs: Any) -> dict[str, Any]:\n ctx = super().get_context_data(**kwargs)\n\n ctx[\"turnstile_form\"] = TurnstileForm(auto_id=False)\n structured_logger.debug(\n \"Added Turnstile form to context.\",\n event_code=\"login_context_turnstile\",\n user=self.request.user,\n )\n return ctx\n\n\n@login_required\n@never_cache\ndef account_letter(request: HttpRequest) -> HttpResponse:\n \"\"\"\n Generate and return a PDF letter summarizing a user's contributions.\n\n This view creates a service letter for the logged-in user, summarizing their\n transcription and review activity. It uses an HTML template rendered with\n contribution data and converts it to a PDF using WeasyPrint.\n\n Requires the user to be authenticated.\n\n Returns:\n response (HttpResponse): A PDF response with content type\n `application/pdf` and a `Content-Disposition` header set to download\n as `letter.pdf`.\n\n Return Behavior:\n - The generated PDF includes:\n - User's name and join date.\n - Total transcriptions and reviews.\n - List of assets the user contributed to.\n \"\"\"\n structured_logger.debug(\n \"Building account letter.\",\n event_code=\"account_letter_start\",\n user=request.user,\n )\n image_url = \"file://{0}/{1}/img/logo.jpg\".format(\n settings.SITE_ROOT_DIR, settings.STATIC_ROOT\n )\n user_profile_activity = UserProfileActivity.objects.filter(user=request.user)\n aggregate_sums = user_profile_activity.aggregate(\n Sum(\"review_count\"), Sum(\"transcribe_count\")\n )\n asset_list = _get_pages(request)\n context = {\n \"user\": request.user,\n \"join_date\": request.user.date_joined,\n \"total_reviews\": aggregate_sums[\"review_count__sum\"],\n \"total_transcriptions\": aggregate_sums[\"transcribe_count__sum\"],\n \"image_url\": image_url,\n \"asset_list\": asset_list,\n }\n template = loader.get_template(\"documents/service_letter.html\")\n text = template.render(context)\n html = HTML(string=text)\n response = HttpResponse(\n content=html.write_pdf(variant=\"pdf/ua-1\"), content_type=\"application/pdf\"\n )\n response[\"Content-Disposition\"] = \"attachment; filename=letter.pdf\"\n structured_logger.debug(\n \"Account letter generated.\",\n event_code=\"account_letter_success\",\n user=request.user,\n total_reviews=aggregate_sums[\"review_count__sum\"],\n total_transcriptions=aggregate_sums[\"transcribe_count__sum\"],\n asset_count=len(asset_list),\n )\n return response\n\n\n@login_required\n@never_cache\ndef get_pages(request: HttpRequest) -> JsonResponse:\n \"\"\"\n Return a paginated and filtered list of the user's contributed assets as HTML.\n\n Retrieves assets the current user has worked on, applies pagination, and\n optionally filters by campaign, activity type, status, and date range. Renders\n the results into a fragment of HTML for use in dynamic page updates.\n\n Requires the user to be authenticated.\n\n Args:\n request (HttpRequest): The request from the authenticated user.\n\n Request Parameters:\n - `page` (int): Page number to display. Example: `2`\n - `campaign` (int): Filter by campaign ID. Example: `17`\n - `status` (list[str]): Filter by asset statuses. Example:\n `[\"in_progress\", \"submitted\"]`\n - `activity` (str): Filter by activity type. Example: `\"transcribe\"`\n - `order_by` (str): Sort order. Example: `\"date-descending\"`\n - `start` (str): Start date in YYYY-MM-DD format. Example: `\"2023-01-01\"`\n - `end` (str): End date in YYYY-MM-DD format. Example: `\"2023-12-31\"`\n\n Returns:\n response (JsonResponse): A JSON object containing rendered HTML for recent\n contributed pages.\n\n Response Format - Success:\n - `content` (str): Rendered HTML for recent pages.\n\n Example:\n ```json\n {\n \"content\": \"...
\"\n }\n ```\n \"\"\"\n tracemalloc.start()\n gc.collect()\n\n structured_logger.debug(\n \"Fetching recent pages.\",\n event_code=\"recent_pages_entry\",\n user=request.user,\n page=request.GET.get(\"page\", \"1\"),\n campaign=request.GET.get(\"campaign\"),\n activity=request.GET.get(\"activity\"),\n )\n asset_list = _get_pages(request)\n\n paginator = Paginator(asset_list, 30) # Show 30 assets per page.\n\n page_number = int(request.GET.get(\"page\", \"1\"))\n context = {\n \"paginator\": paginator,\n \"page_obj\": paginator.get_page(page_number),\n \"is_paginated\": True,\n \"recent_campaigns\": Campaign.objects.filter(\n project__item__asset__in=asset_list.values(\"pk\")\n )\n .distinct()\n .order_by(\"title\")\n .values(\"pk\", \"title\"),\n }\n for param in (\"activity\", \"end\", \"order_by\", \"start\", \"statuses\"):\n context[param] = request.GET.get(param, None)\n campaign = request.GET.get(\"campaign\", None)\n context[\"statuses\"] = request.GET.getlist(\"status\")\n\n if campaign is not None:\n context[\"campaign\"] = Campaign.objects.get(pk=int(campaign))\n\n data = {}\n data[\"content\"] = loader.render_to_string(\n \"fragments/recent-pages.html\", context, request=request\n )\n\n # Capture memory stats\n current, peak = tracemalloc.get_traced_memory()\n current_mb = current / 1024 / 1024\n peak_mb = peak / 1024 / 1024\n\n # For immediate visibility\n structured_logger.info(\n \"Recent pages rendered.\",\n event_code=\"recent_pages_success\",\n user=request.user,\n assets=asset_list.count(),\n num_pages=paginator.num_pages,\n page=page_number,\n memory_current_mb=round(current_mb, 2),\n memory_peak_mb=round(peak_mb, 2),\n )\n\n tracemalloc.stop()\n return JsonResponse(data)\n\n\n@method_decorator(never_cache, name=\"dispatch\")\nclass AccountProfileView(LoginRequiredMixin, FormView, ListView):\n \"\"\"\n Display and update user account profile and contribution history.\n\n Combines functionality from:\n - [LoginRequiredMixin](https://docs.djangoproject.com/en/stable/topics/auth/default/#the-loginrequiredmixin-mixin)\n - [FormView](https://docs.djangoproject.com/en/stable/ref/class-based-views/generic-editing/#formview)\n - [ListView](https://docs.djangoproject.com/en/stable/ref/class-based-views/generic-display/#listview)\n\n Allows authenticated users to:\n - Update their email address and name\n - View a paginated list of assets they have contributed to\n - See aggregate statistics on their transcription and review activity\n\n Email changes require confirmation unless the setting\n `REQUIRE_EMAIL_RECONFIRMATION` is False.\n\n Attributes:\n template_name (str): Template used to render the profile page.\n Example: `\"account/profile.html\"`\n form_class (Form): Form used to update the user's email address.\n Example: `UserProfileForm`\n success_url (str): Redirect URL after successful form submission.\n Example: `\"/accounts/profile/\"`\n allow_empty (bool): Whether to render the page if the user has no\n contributions. Default is `True`\n paginate_by (int): Number of contributed assets to show per page.\n Default is `30`.\n reconfirmation_email_body_template (str): Path to the plain text email\n body template. Example: `\"emails/email_reconfirmation_body.txt\"`\n reconfirmation_email_subject_template (str): Path to the email subject\n template. Example: `\"emails/email_reconfirmation_subject.txt\"`\n\n Returns:\n response (HttpResponse): The rendered profile page with contribution data\n or a redirect to `#account` after successful form submission.\n\n Request Parameters:\n - `page` (int): Page number. Example: `1`\n - `campaign` (int): Campaign filter. Example: `42`\n - `activity` (str): Activity type filter. Example: `\"transcribe\"`\n - `status` (list[str]): Asset statuses. Example: `[\"completed\"]`\n - `start` (str): Start date in YYYY-MM-DD format. Example: `\"2023-01-01\"`\n - `end` (str): End date in YYYY-MM-DD format. Example: `\"2023-12-31\"`\n - `order_by` (str): Sort field. Example: `\"date-descending\"`\n - `tab` (str): Selected tab. Example: `\"account\"`\n \"\"\"\n\n template_name: str = \"account/profile.html\"\n form_class: Type[Form] = UserProfileForm\n success_url = reverse_lazy(\"user-profile\")\n reconfirmation_email_body_template: str = \"emails/email_reconfirmation_body.txt\"\n reconfirmation_email_subject_template: str = (\n \"emails/email_reconfirmation_subject.txt\"\n )\n\n # This view will list the assets which the user has contributed to\n # along with their most recent action on each asset. This will be\n # presented in the template as a standard paginated list of Asset\n # instances with annotations\n allow_empty: bool = True\n paginate_by: int = 30\n\n def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:\n structured_logger.debug(\n \"Profile POST received.\",\n event_code=\"profile_post_entry\",\n user=request.user,\n )\n self.object_list = self.get_queryset()\n if \"submit_name\" in request.POST:\n form = UserNameForm(request.POST)\n if form.is_valid():\n user = ConcordiaUser.objects.get(id=request.user.id)\n user.first_name = form.cleaned_data[\"first_name\"]\n user.last_name = form.cleaned_data[\"last_name\"]\n user.save()\n structured_logger.debug(\n \"Updated profile name.\",\n event_code=\"profile_name_updated\",\n user=request.user,\n )\n return redirect(\"user-profile\")\n else:\n return super().post(request, *args, **kwargs)\n\n def get_queryset(self) -> Any:\n structured_logger.debug(\n \"Fetching pages for profile.\",\n event_code=\"profile_get_queryset\",\n user=self.request.user,\n )\n return _get_pages(self.request)\n\n def get_context_data(self, *args: Any, **kwargs: Any) -> dict[str, Any]:\n ctx = super().get_context_data(*args, **kwargs)\n\n page = self.request.GET.get(\"page\", None)\n campaign = self.request.GET.get(\"campaign\", None)\n activity = self.request.GET.get(\"activity\", None)\n status_list = self.request.GET.getlist(\"status\")\n start = self.request.GET.get(\"start\", None)\n end = self.request.GET.get(\"end\", None)\n order_by = self.request.GET.get(\"order_by\", None)\n if any([activity, campaign, page, status_list, start, end, order_by]):\n ctx[\"active_tab\"] = \"recent\"\n if status_list:\n ctx[\"status_list\"] = status_list\n ctx[\"order_by\"] = self.request.GET.get(\"order_by\", \"date-descending\")\n elif \"active_tab\" not in ctx:\n ctx[\"active_tab\"] = self.request.GET.get(\"tab\", \"contributions\")\n ctx[\"activity\"] = activity\n if end is not None:\n ctx[\"end\"] = end\n ctx[\"order_by\"] = order_by\n if start is not None:\n ctx[\"start\"] = start\n\n ctx[\"valid\"] = self.request.session.pop(\"valid\", None)\n\n user = self.request.user\n concordia_user = ConcordiaUser.objects.get(id=user.id)\n user_profile_activity = UserProfileActivity.objects.filter(user=user).order_by(\n \"campaign__title\"\n )\n ctx[\"user_profile_activity\"] = user_profile_activity\n\n aggregate_sums = user_profile_activity.aggregate(\n Sum(\"review_count\"), Sum(\"transcribe_count\"), Sum(\"asset_count\")\n )\n ctx[\"totalReviews\"] = aggregate_sums[\"review_count__sum\"]\n ctx[\"totalTranscriptions\"] = aggregate_sums[\"transcribe_count__sum\"]\n ctx[\"pages_worked_on\"] = aggregate_sums[\"asset_count__sum\"]\n if ctx[\"totalReviews\"] is not None:\n ctx[\"totalCount\"] = ctx[\"totalReviews\"] + ctx[\"totalTranscriptions\"]\n ctx[\"unconfirmed_email\"] = concordia_user.get_email_for_reconfirmation()\n ctx[\"name_form\"] = UserNameForm()\n structured_logger.debug(\n \"Profile context ready.\",\n event_code=\"profile_context_ready\",\n user=self.request.user,\n total_reviews=ctx[\"totalReviews\"],\n total_transcriptions=ctx[\"totalTranscriptions\"],\n pages_worked=ctx[\"pages_worked_on\"],\n )\n return ctx\n\n def get_initial(self) -> dict[str, Any]:\n initial = super().get_initial()\n initial[\"email\"] = self.request.user.email\n return initial\n\n def get_form_kwargs(self) -> dict[str, Any]:\n # We'll expose the request object to the form so we can validate that an\n # email is not in use:\n kwargs = super().get_form_kwargs()\n kwargs[\"request\"] = self.request\n return kwargs\n\n def form_valid(self, form: Form) -> HttpResponse:\n user = self.request.user\n new_email = form.cleaned_data[\"email\"]\n structured_logger.info(\n \"Profile email update submitted.\",\n event_code=\"profile_email_update\",\n user=user,\n require_reconfirm=settings.REQUIRE_EMAIL_RECONFIRMATION,\n )\n # This is annoying, but there's no better way to get the proxy model here\n # without being hacky (changing user.__class__ directly.)\n # Every method (such as using a user profile) would incur the same\n # database request.\n concordia_user = ConcordiaUser.objects.get(id=user.id)\n if settings.REQUIRE_EMAIL_RECONFIRMATION:\n concordia_user.set_email_for_reconfirmation(new_email)\n structured_logger.debug(\n \"Email set for reconfirmation.\",\n event_code=\"email_reconfirm_set\",\n user=user,\n new_email=new_email,\n )\n self.send_reconfirmation_email(concordia_user)\n else:\n concordia_user.email = new_email\n concordia_user.full_clean()\n concordia_user.save()\n concordia_user.delete_email_for_reconfirmation()\n structured_logger.debug(\n \"Email updated without reconfirmation.\",\n event_code=\"email_updated_no_reconfirm\",\n user=user,\n new_email=new_email,\n )\n\n self.request.session[\"valid\"] = True\n\n return super().form_valid(form)\n\n def form_invalid(self, form: Form) -> HttpResponse:\n structured_logger.debug(\n \"Profile form invalid.\",\n event_code=\"profile_form_invalid\",\n user=self.request.user,\n )\n self.request.session[\"valid\"] = False\n return self.render_to_response(\n self.get_context_data(form=form, active_tab=\"account\")\n )\n\n def get_success_url(self) -> str:\n # automatically open the Account Settings tab\n return \"{}#account\".format(super().get_success_url())\n\n def get_reconfirmation_email_context(self, confirmation_key: str) -> dict[str, Any]:\n return {\n \"confirmation_key\": confirmation_key,\n \"expiration_days\": settings.EMAIL_RECONFIRMATION_DAYS,\n \"site\": get_current_site(self.request),\n }\n\n def send_reconfirmation_email(self, user: ConcordiaUser) -> None:\n confirmation_key = user.get_email_reconfirmation_key()\n context = self.get_reconfirmation_email_context(confirmation_key)\n context[\"user\"] = user\n subject = render_to_string(\n template_name=self.reconfirmation_email_subject_template,\n context=context,\n request=self.request,\n )\n # Ensure subject is a single line\n subject = \"\".join(subject.splitlines())\n message = render_to_string(\n template_name=self.reconfirmation_email_body_template,\n context=context,\n request=self.request,\n )\n try:\n structured_logger.info(\n \"Sending reconfirmation email.\",\n event_code=\"email_reconfirm_send_start\",\n user=user,\n email=user.get_email_for_reconfirmation(),\n )\n send_mail(\n subject,\n message=message,\n from_email=settings.DEFAULT_FROM_EMAIL,\n recipient_list=[user.get_email_for_reconfirmation()],\n )\n structured_logger.debug(\n \"Reconfirmation email sent.\",\n event_code=\"email_reconfirm_send_success\",\n user=user,\n email=user.get_email_for_reconfirmation(),\n )\n except SMTPException:\n logger.exception(\n \"Unable to send email reconfirmation to %s\",\n user.get_email_for_reconfirmation(),\n )\n structured_logger.exception(\n \"Reconfirmation email send failed.\",\n event_code=\"email_reconfirm_send_failed\",\n reason=\"SMTPException\",\n reason_code=\"smtp_error\",\n user=user,\n email=user.get_email_for_reconfirmation(),\n )\n messages.error(\n self.request,\n _(\"Email confirmation could not be sent.\"),\n )\n\n\n@method_decorator(never_cache, name=\"dispatch\")\nclass AccountDeletionView(LoginRequiredMixin, FormView):\n \"\"\"\n Handle user-initiated account deletion.\n\n Extends:\n - [LoginRequiredMixin](https://docs.djangoproject.com/en/stable/topics/auth/default/#the-loginrequiredmixin-mixin)\n - [FormView](https://docs.djangoproject.com/en/stable/ref/class-based-views/generic-editing/#formview)\n\n Provides a confirmation form for deleting the user's account. If the user has\n contributed transcriptions, their data is anonymized instead of being deleted.\n Otherwise, the account is fully removed. A confirmation email is sent to the\n user's address before deletion. After deletion, the user is logged out.\n\n Requires the user to be authenticated.\n\n Attributes:\n template_name (str): Template used to render the confirmation form.\n Example: `\"account/account_deletion.html\"`\n form_class (Form): Form used to confirm account deletion.\n Example: `AccountDeletionForm`\n success_url (str): URL to redirect to after deletion.\n Example: `\"/\"`\n email_body_template (str): Template for the body of the confirmation email.\n Example: `\"emails/delete_account_body.txt\"`\n email_subject_template (str): Template for the subject of the confirmation\n email. Example: `\"emails/delete_account_subject.txt\"`\n\n Returns:\n response (HttpResponse): A redirect to the homepage after deletion, or a\n rendered form with errors if validation fails.\n\n Return Behavior:\n - If the user confirms deletion and has transcriptions: anonymizes their\n account and logs them out.\n - If the user has no transcriptions: deletes the account entirely and logs\n them out.\n - If the form is invalid: re-renders the confirmation form with errors.\n \"\"\"\n\n template_name: str = \"account/account_deletion.html\"\n form_class: Type[Form] = AccountDeletionForm\n success_url: str = reverse_lazy(\"homepage\")\n email_body_template: str = \"emails/delete_account_body.txt\"\n email_subject_template: str = \"emails/delete_account_subject.txt\"\n\n def get_form_kwargs(self) -> dict[str, Any]:\n # We expose the request object to the form so we can use it\n # to log the user out after deletion\n kwargs = super().get_form_kwargs()\n kwargs[\"request\"] = self.request\n return kwargs\n\n def form_valid(self, form: Form) -> HttpResponse:\n structured_logger.info(\n \"Account deletion confirmed.\",\n event_code=\"account_delete_confirmed\",\n user=form.request.user,\n )\n self.delete_user(form.request.user, form.request)\n return super().form_valid(form)\n\n def delete_user(self, user: ConcordiaUser, request: HttpRequest) -> None:\n logger.info(\"Deletion request for %s\", user)\n structured_logger.info(\n \"Processing account deletion.\",\n event_code=\"account_delete_start\",\n user=user,\n )\n email = user.email\n if user.transcription_set.exists():\n logger.info(\"Anonymizing %s\", user)\n structured_logger.info(\n \"Anonymizing user account.\",\n event_code=\"account_anonymize\",\n user=user,\n )\n user.username = \"Anonymized %s\" % uuid.uuid4()\n user.first_name = \"\"\n user.last_name = \"\"\n user.email = \"\"\n user.set_unusable_password()\n user.is_staff = False\n user.is_superuser = False\n user.is_active = False\n user.save()\n else:\n logger.info(\"Deleting %s\", user)\n structured_logger.info(\n \"Deleting user account.\",\n event_code=\"account_delete\",\n user=user,\n )\n user.delete()\n self.send_deletion_email(email)\n logout(request)\n structured_logger.info(\n \"Account deletion complete.\",\n event_code=\"account_delete_complete\",\n user=user,\n )\n\n def send_deletion_email(self, email: str) -> None:\n context = {}\n subject = render_to_string(\n template_name=self.email_subject_template,\n context=context,\n request=self.request,\n )\n # Ensure subject is a single line\n subject = \"\".join(subject.splitlines())\n message = render_to_string(\n template_name=self.email_body_template,\n context=context,\n request=self.request,\n )\n try:\n structured_logger.info(\n \"Sending deletion email.\",\n event_code=\"account_delete_email_send_start\",\n user=self.request.user,\n email=email,\n )\n send_mail(\n subject,\n message=message,\n from_email=settings.DEFAULT_FROM_EMAIL,\n recipient_list=[email],\n )\n structured_logger.debug(\n \"Deletion email sent.\",\n event_code=\"account_delete_email_send_success\",\n user=self.request.user,\n email=email,\n )\n except SMTPException:\n logger.exception(\n \"Unable to send account deletion email to %s\",\n email,\n )\n structured_logger.exception(\n \"Deletion email send failed.\",\n event_code=\"account_delete_email_send_failed\",\n reason=\"SMTPException\",\n reason_code=\"smtp_error\",\n user=self.request.user,\n email=email,\n )\n messages.error(\n self.request,\n _(\"Email confirmation of deletion could not be sent.\"),\n )\n\n\nclass EmailReconfirmationView(TemplateView):\n \"\"\"\n Handle email reconfirmation via a signed URL token.\n\n Extends:\n - [TemplateView](https://docs.djangoproject.com/en/stable/ref/class-based-views/base/#templateview)\n\n Validates a confirmation key sent to the user's new email address during\n an address change. If valid and not expired, applies the email update. If\n invalid, expired or mismatched, renders an error message.\n\n Attributes:\n template_name (str): Template rendered if the confirmation fails.\n Example: `\"account/email_reconfirmation_failed.html\"`\n success_url (str): URL to redirect to on success.\n Example: `\"/accounts/profile/#account\"`\n BAD_USERNAME_MESSAGE (str): Error if the user account cannot be found.\n BAD_EMAIL_MESSAGE (str): Error if the email does not match expectations.\n EXPIRED_MESSAGE (str): Error if the key is expired.\n INVALID_KEY_MESSAGE (str): Error if the key signature is invalid.\n\n Returns:\n response (HttpResponse): Redirects to the profile page with `#account`\n on success, or renders the failure template with error details.\n\n Request Parameters:\n confirmation_key (str): A signed token containing the username and new\n email. Example: `\"ZHVtbXl1c2VyOnNvbWVvbmVAZXhhbXBsZS5jb20=\"`\n \"\"\"\n\n success_url = reverse_lazy(\"user-profile\")\n template_name = \"account/email_reconfirmation_failed.html\"\n\n BAD_USERNAME_MESSAGE: str = _(\"The account you attempted to confirm is invalid.\")\n BAD_EMAIL_MESSAGE: str = _(\"The email you attempted to confirm is invalid.\")\n EXPIRED_MESSAGE: str = _(\n \"The confirmation key you provided is expired. Email confirmation links \"\n \"expire after 7 days. If your key is expired, you will need to re-enter \"\n \"your new email address\"\n )\n INVALID_KEY_MESSAGE: str = _(\n \"The confirmation key you provided is invalid. Email confirmation links \"\n \"expire after 7 days. If your key is expired, you will need to re-enter \"\n \"your new email address.\"\n )\n\n def get_success_url(self) -> str:\n return \"{}#account\".format(self.success_url)\n\n def get(self, *args: Any, **kwargs: Any) -> HttpResponse:\n extra_context = {}\n try:\n structured_logger.debug(\n \"Email reconfirmation GET received.\",\n event_code=\"email_reconfirm_entry\",\n )\n self.confirm(*args, **kwargs)\n except ValidationError as exc:\n structured_logger.warning(\n \"Email reconfirmation failed.\",\n event_code=\"email_reconfirm_failed\",\n reason=str(exc.message),\n reason_code=str(exc.code),\n )\n extra_context[\"reconfirmation_error\"] = {\n \"message\": exc.message,\n \"code\": exc.code,\n \"params\": exc.params,\n }\n context_data = self.get_context_data()\n context_data.update(extra_context)\n return self.render_to_response(context_data, status=403)\n else:\n structured_logger.debug(\n \"Email reconfirmation ok.\",\n event_code=\"email_reconfirm_success\",\n )\n return HttpResponseRedirect(self.get_success_url())\n\n def confirm(self, *args: Any, **kwargs: Any) -> ConcordiaUser:\n username, email = self.validate_key(kwargs.get(\"confirmation_key\"))\n user = self.get_user(username)\n if not user.validate_reconfirmation_email(email):\n raise ValidationError(self.BAD_EMAIL_MESSAGE, code=\"bad_email\") from None\n try:\n user.email = email\n user.full_clean()\n except ValidationError:\n raise ValidationError(self.BAD_EMAIL_MESSAGE, code=\"bad_email\") from None\n user.save()\n user.delete_email_for_reconfirmation()\n structured_logger.info(\n \"Email reconfirmed and applied.\",\n event_code=\"email_reconfirm_applied\",\n user=user,\n new_email=email,\n )\n return user\n\n def validate_key(self, confirmation_key: str) -> tuple[str, str]:\n try:\n context = signing.loads(\n confirmation_key, max_age=settings.EMAIL_RECONFIRMATION_TIMEOUT\n )\n return context[\"username\"], context[\"email\"]\n except signing.SignatureExpired as exc:\n raise ValidationError(self.EXPIRED_MESSAGE, code=\"expired\") from exc\n except signing.BadSignature as exc:\n raise ValidationError(\n self.INVALID_KEY_MESSAGE,\n code=\"invalid_key\",\n params={\"confirmation_key\": confirmation_key},\n ) from exc\n\n def get_user(self, username: str) -> ConcordiaUser:\n try:\n user = ConcordiaUser.objects.get(username=username)\n structured_logger.debug(\n \"Loaded user for reconfirmation.\",\n event_code=\"email_reconfirm_user_loaded\",\n user=user,\n )\n return user\n except ConcordiaUser.DoesNotExist as exc:\n structured_logger.warning(\n \"User not found for reconfirmation.\",\n event_code=\"email_reconfirm_user_missing\",\n reason=\"User does not exist\",\n reason_code=\"user_missing\",\n )\n raise ValidationError(\n self.BAD_USERNAME_MESSAGE, code=\"bad_username\"\n ) from exc\n"
},
{
"path": "concordia/views/ajax.py",
"content": "import logging\nimport re\nfrom time import time\nfrom typing import Union\n\nfrom django.conf import settings\nfrom django.contrib.auth.decorators import login_required\nfrom django.contrib.messages import get_messages\nfrom django.core.exceptions import ValidationError\nfrom django.db import connection\nfrom django.db.transaction import atomic\nfrom django.http import HttpRequest, HttpResponse, JsonResponse\nfrom django.shortcuts import get_object_or_404\nfrom django.urls import reverse\nfrom django.utils.timezone import now\nfrom django.views.decorators.cache import cache_control, never_cache\nfrom django.views.decorators.csrf import csrf_exempt\nfrom django.views.decorators.http import require_POST\nfrom django_ratelimit.decorators import ratelimit\n\nfrom concordia.exceptions import RateLimitExceededError\nfrom concordia.logging import ConcordiaLogger\nfrom concordia.models import (\n Asset,\n AssetTranscriptionReservation,\n ConcordiaUser,\n Tag,\n Transcription,\n UserAssetTagCollection,\n)\nfrom concordia.signals.signals import (\n reservation_obtained,\n reservation_released,\n)\nfrom concordia.utils import (\n get_anonymous_user,\n get_or_create_reservation_token,\n)\nfrom concordia.utils.constants import MESSAGE_LEVEL_NAMES, URL_REGEX\nfrom configuration.utils import configuration_value\nfrom exporter.utils import remove_unacceptable_characters\n\nfrom .decorators import reserve_rate, validate_anonymous_user\n\nlogger = logging.getLogger(__name__)\nstructured_logger = ConcordiaLogger.get_logger(__name__)\n\n\n@cache_control(private=True, max_age=settings.DEFAULT_PAGE_TTL)\n@csrf_exempt\ndef ajax_session_status(request: HttpRequest) -> JsonResponse:\n \"\"\"\n Return a JSON object describing the authenticated session state.\n\n If the user is authenticated, this includes a truncated username and\n navigational links to profile, logout, and admin pages (if staff or superuser).\n If the user is anonymous, returns an empty dictionary.\n\n Args:\n request (HttpRequest): The HTTP request initiating the session status check.\n\n Returns:\n response (JsonResponse): A dictionary containing either session information\n or empty data.\n\n Response Format - Success:\n - `username` (str): The first 15 characters of the user's username.\n - `links` (list[dict]): A list of links relevant to the user's session.\n - `title` (str): The label for the link (e.g., \"Profile\", \"Logout\").\n - `url` (str): The absolute URL for the link.\n\n Example:\n ```json\n // If the user is authenticated:\n {\n \"username\": \"johndoe\",\n \"links\": [\n {\"title\": \"Profile\", \"url\": \"https://example.com/accounts/profile/\"},\n {\"title\": \"Logout\", \"url\": \"https://example.com/accounts/logout/\"}\n ]\n }\n\n // If the user is anonymous:\n {}\n ```\n \"\"\"\n user = request.user\n if user.is_anonymous:\n res = {}\n else:\n links = [\n {\n \"title\": \"Profile\",\n \"type\": \"link\",\n \"url\": request.build_absolute_uri(reverse(\"user-profile\")),\n }\n ]\n if user.is_superuser or user.is_staff:\n links.append(\n {\n \"title\": \"Admin Area\",\n \"type\": \"link\",\n \"url\": request.build_absolute_uri(reverse(\"admin:index\")),\n }\n )\n links.append(\n {\n \"title\": \"Logout\",\n \"type\": \"post\",\n \"url\": request.build_absolute_uri(reverse(\"logout\")),\n \"fields\": {\"next\": \"/\"},\n }\n )\n\n res = {\"username\": user.username[:15], \"links\": links}\n\n return JsonResponse(res)\n\n\n@never_cache\n@login_required\n@csrf_exempt\ndef ajax_messages(request: HttpRequest) -> JsonResponse:\n \"\"\"\n Return a JSON object containing the user's queued messages.\n\n Retrieves Django messages for the current request and formats them\n as a list of dictionaries, each containing the message text and its\n severity level.\n\n Requires the user to be authenticated.\n\n Args:\n request (HttpRequest): The request from the authenticated user.\n\n Returns:\n response (JsonResponse): A dictionary with a `messages` field containing\n a list of message entries.\n\n Response Format - Success:\n - `messages` (list[dict]): A list of user-visible messages.\n - `level` (str): The severity level of the message\n (e.g., \"info\", \"warning\", \"error\").\n - `message` (str): The text content of the message.\n\n Example:\n ```json\n {\n \"messages\": [\n {\"level\": \"info\", \"message\": \"You have been logged out.\"},\n {\"level\": \"warning\", \"message\": \"Your session is about to expire.\"}\n ]\n }\n ```\n \"\"\"\n return JsonResponse(\n {\n \"messages\": [\n {\"level\": MESSAGE_LEVEL_NAMES[i.level], \"message\": i.message}\n for i in get_messages(request)\n ]\n }\n )\n\n\ndef get_transcription_superseded(\n asset: Asset, supersedes_pk: Union[int, str, None]\n) -> Union[Transcription, JsonResponse, None]:\n \"\"\"\n Determine the superseded transcription, if any, for a new transcription.\n\n If a valid `supersedes_pk` is provided, returns the corresponding transcription\n unless it has already been superseded. If no `supersedes_pk` is provided,\n checks whether the asset already has an open transcription.\n\n This helper may return an error response to be passed directly to the client,\n or a transcription object used when saving a new one.\n\n Args:\n asset (Asset): The asset the transcription is associated with.\n supersedes_pk (int or str or None): The primary key of the transcription\n being superseded, or `None` if this is the first transcription.\n\n Returns:\n response (Transcription or JsonResponse or None): A valid transcription,\n an error response, or `None`.\n\n Return Behavior:\n - If a valid transcription is found, a `Transcription` object is returned.\n - If the request is invalid or the transcription has already been superseded,\n a `JsonResponse` with an error is returned.\n - If there is no previous transcription to supersede, `None` is returned.\n\n Response Format - Error:\n - `error` (str): Explanation of why the transcription cannot be created.\n - \"An open transcription already exists\"\n - \"This transcription has been superseded\"\n - \"Invalid supersedes value\"\n\n Example:\n ```json\n {\n \"error\": \"An open transcription already exists\"\n }\n ```\n \"\"\"\n structured_logger.info(\n \"Checking for superseded transcription.\",\n event_code=\"transcription_supersede_check_start\",\n asset=asset,\n supersedes_pk=supersedes_pk,\n )\n if not supersedes_pk:\n if asset.transcription_set.filter(supersedes=None).exists():\n structured_logger.warning(\n \"Open transcription already exists for asset.\",\n event_code=\"transcription_supersede_check_failed\",\n reason=\"An open transcription already exists\",\n reason_code=\"already_exists\",\n asset=asset,\n )\n return JsonResponse(\n {\"error\": \"An open transcription already exists\"}, status=409\n )\n else:\n superseded = None\n else:\n try:\n if asset.transcription_set.filter(supersedes=supersedes_pk).exists():\n structured_logger.warning(\n \"Transcription already superseded.\",\n event_code=\"transcription_supersede_check_failed\",\n reason=\"This transcription has been superseded\",\n reason_code=\"already_superseded\",\n asset=asset,\n supersedes_pk=supersedes_pk,\n )\n return JsonResponse(\n {\"error\": \"This transcription has been superseded\"}, status=409\n )\n\n try:\n superseded = asset.transcription_set.get(pk=supersedes_pk)\n except Transcription.DoesNotExist:\n structured_logger.warning(\n \"Supersedes transcription not found.\",\n event_code=\"transcription_supersede_check_failed\",\n reason=\"Invalid supersedes value\",\n reason_code=\"not_found\",\n asset=asset,\n supersedes_pk=supersedes_pk,\n )\n return JsonResponse({\"error\": \"Invalid supersedes value\"}, status=400)\n except ValueError:\n structured_logger.warning(\n \"Invalid supersedes value (non-integer).\",\n event_code=\"transcription_supersede_check_failed\",\n reason=\"Supersedes value must be an integer\",\n reason_code=\"invalid_pk_format\",\n asset=asset,\n supersedes_pk=supersedes_pk,\n )\n return JsonResponse({\"error\": \"Invalid supersedes value\"}, status=400)\n structured_logger.info(\n \"Superseded transcription found.\",\n event_code=\"transcription_supersede_check_success\",\n asset=asset,\n supersedes_pk=supersedes_pk,\n )\n return superseded\n\n\n@require_POST\n@login_required\n@atomic\n@ratelimit(key=\"header:cf-connecting-ip\", rate=\"1/m\", block=settings.RATELIMIT_BLOCK)\ndef generate_ocr_transcription(\n request: HttpRequest, *, asset_pk: Union[int, str]\n) -> JsonResponse:\n \"\"\"\n Create and save a new OCR-generated transcription for an asset.\n\n If no prior transcription exists, creates a blank transcription to serve as\n the superseded record. Otherwise, the specified previous transcription is\n superseded by the new OCR transcription.\n\n Requires the user to be authenticated.\n\n Request Parameters:\n - `supersedes` (int or str, optional): The ID of the transcription being\n superseded.\n - `language` (str, optional): The language code to influence OCR output.\n\n Returns:\n response (JsonResponse): A dictionary describing the new transcription\n and asset status.\n\n Response Format - Success:\n - `id` (int): ID of the new transcription.\n - `sent` (float): UNIX timestamp when the transcription was created.\n - `submissionUrl` (str): URL to submit the transcription.\n - `text` (str): The OCR-generated transcription content.\n - `asset` (dict):\n - `id` (int): ID of the associated asset.\n - `status` (str): Current transcription status.\n - `contributors` (int): Number of users who have contributed.\n - `undo_available` (bool): Whether the user can roll back this transcription.\n - `redo_available` (bool): Whether the user can roll forward to another version.\n\n Example:\n ```json\n {\n \"id\": 123,\n \"sent\": 1716294920.927134,\n \"submissionUrl\": \"/transcriptions/123/submit/\",\n \"text\": \"Detected OCR content...\",\n \"asset\": {\n \"id\": 456,\n \"status\": \"in_progress\",\n \"contributors\": 2\n },\n \"undo_available\": true,\n \"redo_available\": false\n }\n ```\n \"\"\"\n asset = get_object_or_404(Asset, pk=asset_pk)\n user = request.user\n\n supersedes_pk = request.POST.get(\"supersedes\")\n language = request.POST.get(\"language\", None)\n structured_logger.info(\n \"Starting OCR transcription generation.\",\n event_code=\"ocr_generation_start\",\n user=user,\n asset=asset,\n supersedes_pk=supersedes_pk,\n language=language,\n )\n superseded = get_transcription_superseded(asset, supersedes_pk)\n if superseded:\n # If superseded is an HttpResponse, that means\n # this transcription has already been superseded, so\n # we won't run OCR and instead send back an error\n # Otherwise, we just have thr transcription the OCR\n # is gong to supersede, so we can continue\n if isinstance(superseded, HttpResponse):\n structured_logger.warning(\n \"OCR generation aborted: superseded transcription is invalid.\",\n event_code=\"ocr_generation_aborted\",\n reason=\"Superseded transcription is invalid\",\n reason_code=\"superseded_invalid\",\n user=user,\n asset=asset,\n )\n return superseded\n else:\n # This means this is the first transcription on this asset.\n # To enable undoing of the OCR transcription, we create\n # an empty transcription for the OCR transcription to supersede\n structured_logger.info(\n \"No existing transcription; creating empty one for OCR supersession.\",\n event_code=\"ocr_blank_supersede\",\n user=user,\n asset=asset,\n )\n superseded = Transcription(\n asset=asset,\n user=get_anonymous_user(),\n text=\"\",\n )\n superseded.full_clean()\n superseded.save()\n structured_logger.info(\n \"Blank superseded transcription created for OCR.\",\n event_code=\"ocr_blank_transcription_created\",\n user=user,\n transcription=superseded,\n )\n\n transcription_text = asset.get_ocr_transcript(language)\n transcription = Transcription(\n asset=asset,\n user=user,\n supersedes=superseded,\n text=transcription_text,\n ocr_generated=True,\n ocr_originated=True,\n )\n transcription.full_clean()\n transcription.save()\n\n structured_logger.info(\n \"OCR transcription successfully created.\",\n event_code=\"ocr_generation_success\",\n user=user,\n transcription=transcription,\n )\n\n return JsonResponse(\n {\n \"id\": transcription.pk,\n \"sent\": time(),\n \"submissionUrl\": reverse(\"submit-transcription\", args=(transcription.pk,)),\n \"text\": transcription.text,\n \"asset\": {\n \"id\": transcription.asset.id,\n \"status\": transcription.asset.transcription_status,\n \"contributors\": transcription.asset.get_contributor_count(),\n },\n \"undo_available\": asset.can_rollback()[0],\n \"redo_available\": asset.can_rollforward()[0],\n },\n status=201,\n )\n\n\n@require_POST\n@validate_anonymous_user\n@atomic\n@ratelimit(key=\"header:cf-connecting-ip\", rate=\"1/m\", block=settings.RATELIMIT_BLOCK)\ndef rollback_transcription(\n request: HttpRequest, *, asset_pk: Union[int, str]\n) -> JsonResponse:\n \"\"\"\n Perform a rollback on the latest transcription for the given asset.\n\n Restores the asset's transcription to the previous version in its history.\n If rollback is not possible (e.g., no prior version exists), returns an error.\n\n Anonymous users are supported and handled via `get_anonymous_user()`. The caller\n must be validated via `validate_anonymous_user`.\n\n Args:\n request (HttpRequest): The POST request to initiate rollback.\n asset_pk (int or str): The primary key of the asset being rolled back.\n\n Returns:\n response (JsonResponse): A dictionary containing the restored transcription\n and asset status, or an error response if rollback fails.\n\n Response Format - Success:\n - `id` (int): ID of the restored transcription.\n - `sent` (float): UNIX timestamp of the response.\n - `submissionUrl` (str): URL to submit the transcription.\n - `text` (str): The restored transcription text.\n - `asset` (dict):\n - `id` (int): ID of the asset.\n - `status` (str): Current transcription status.\n - `contributors` (int): Number of users who contributed.\n - `message` (str): Confirmation message.\n - `undo_available` (bool): Whether rollback is possible again.\n - `redo_available` (bool): Whether rollforward is now available.\n\n Response Format - Error:\n - `error` (str): Explanation of the failure.\n - \"No previous transcription available\"\n\n Example:\n ```json\n {\n \"id\": 123,\n \"sent\": 1716295121.113204,\n \"submissionUrl\": \"/transcriptions/123/submit/\",\n \"text\": \"Previous transcription text\",\n \"asset\": {\n \"id\": 456,\n \"status\": \"in_progress\",\n \"contributors\": 1\n },\n \"message\": \"Successfully rolled back transcription to previous version\",\n \"undo_available\": false,\n \"redo_available\": true\n }\n ```\n \"\"\"\n asset = get_object_or_404(Asset, pk=asset_pk)\n\n if request.user.is_anonymous:\n user = get_anonymous_user()\n else:\n user = request.user\n\n try:\n transcription = asset.rollback_transcription(user)\n except ValueError as e:\n logger.exception(\"No previous transcription available for rollback\", exc_info=e)\n structured_logger.warning(\n \"Rollback failed: no previous transcription to revert to.\",\n event_code=\"rollback_failed\",\n reason_code=\"no_valid_target\",\n reason=str(e),\n asset=asset,\n user=user,\n )\n return JsonResponse(\n {\"error\": \"No previous transcription available\"}, status=400\n )\n\n structured_logger.info(\n \"Rollback successfully performed.\",\n event_code=\"rollback_success\",\n user=user,\n transcription=transcription,\n )\n\n return JsonResponse(\n {\n \"id\": transcription.pk,\n \"sent\": time(),\n \"submissionUrl\": reverse(\"submit-transcription\", args=(transcription.pk,)),\n \"text\": transcription.text,\n \"asset\": {\n \"id\": transcription.asset.id,\n \"status\": transcription.asset.transcription_status,\n \"contributors\": transcription.asset.get_contributor_count(),\n },\n \"message\": \"Successfully rolled back transcription to previous version\",\n \"undo_available\": transcription.asset.can_rollback()[0],\n \"redo_available\": transcription.asset.can_rollforward()[0],\n },\n status=201,\n )\n\n\n@require_POST\n@validate_anonymous_user\n@atomic\n@ratelimit(key=\"header:cf-connecting-ip\", rate=\"1/m\", block=settings.RATELIMIT_BLOCK)\ndef rollforward_transcription(\n request: HttpRequest, *, asset_pk: Union[int, str]\n) -> JsonResponse:\n \"\"\"\n Perform a rollforward to the transcription previously replaced by a rollback.\n\n Restores the asset's transcription to the next version in its history,\n if a valid rollforward target exists. If not, returns an error response.\n\n Anonymous users are supported and handled via `get_anonymous_user()`. The caller\n must be validated via `validate_anonymous_user`.\n\n Args:\n request (HttpRequest): The POST request to initiate rollforward.\n asset_pk (int or str): The primary key of the asset being rolled forward.\n\n Returns:\n response (JsonResponse): A dictionary containing the restored transcription\n and asset status, or an error response if rollforward fails.\n\n Response Format - Success:\n - `id` (int): ID of the restored transcription.\n - `sent` (float): UNIX timestamp of the response.\n - `submissionUrl` (str): URL to submit the transcription.\n - `text` (str): The restored transcription text.\n - `asset` (dict):\n - `id` (int): ID of the asset.\n - `status` (str): Current transcription status.\n - `contributors` (int): Number of users who contributed.\n - `message` (str): Confirmation message.\n - `undo_available` (bool): Whether rollback is now possible.\n - `redo_available` (bool): Whether another rollforward is possible.\n\n Response Format - Error:\n - `error` (str): Explanation of the failure.\n - \"No transcription to restore\"\n\n Example:\n ```json\n {\n \"id\": 124,\n \"sent\": 1716295243.029184,\n \"submissionUrl\": \"/transcriptions/124/submit/\",\n \"text\": \"Next transcription text\",\n \"asset\": {\n \"id\": 456,\n \"status\": \"in_progress\",\n \"contributors\": 1\n },\n \"message\": \"Successfully restored transcription to next version\",\n \"undo_available\": true,\n \"redo_available\": false\n }\n ```\n \"\"\"\n asset = get_object_or_404(Asset, pk=asset_pk)\n\n if request.user.is_anonymous:\n user = get_anonymous_user()\n else:\n user = request.user\n\n try:\n transcription = asset.rollforward_transcription(user)\n except ValueError as e:\n logger.exception(\"No transcription available for rollforward\", exc_info=e)\n structured_logger.warning(\n \"Rollforward failed: no transcription available to restore.\",\n event_code=\"rollforward_failed\",\n reason_code=\"no_valid_target\",\n reason=str(e),\n asset=asset,\n user=user,\n )\n return JsonResponse({\"error\": \"No transcription to restore\"}, status=400)\n\n structured_logger.info(\n \"Rollforward successfully performed.\",\n event_code=\"rollforward_success\",\n user=user,\n transcription=transcription,\n )\n\n return JsonResponse(\n {\n \"id\": transcription.pk,\n \"sent\": time(),\n \"submissionUrl\": reverse(\"submit-transcription\", args=(transcription.pk,)),\n \"text\": transcription.text,\n \"asset\": {\n \"id\": transcription.asset.id,\n \"status\": transcription.asset.transcription_status,\n \"contributors\": transcription.asset.get_contributor_count(),\n },\n \"message\": \"Successfully restored transcription to next version\",\n \"undo_available\": transcription.asset.can_rollback()[0],\n \"redo_available\": transcription.asset.can_rollforward()[0],\n },\n status=201,\n )\n\n\n@require_POST\n@validate_anonymous_user\n@atomic\ndef save_transcription(\n request: HttpRequest, *, asset_pk: Union[int, str]\n) -> JsonResponse:\n \"\"\"\n Save a transcription draft for a given asset.\n\n Validates the transcription text for disallowed content (e.g., URLs).\n Non-printable characters are automatically removed before saving,\n using the shared exporter sanitization utilities. The view also checks\n for supersession rules. If valid, it creates and saves a new\n transcription associated with the current or anonymous user.\n\n Request Parameters:\n - `text` (str): The transcription text.\n - `supersedes` (int or str, optional): The ID of the transcription\n being superseded. Example: `\"123\"`\n\n Returns:\n response (JsonResponse): A dictionary describing the saved transcription\n and asset status, or an error response if validation fails.\n\n Response Format - Success:\n - `id` (int): ID of the saved transcription.\n - `sent` (float): UNIX timestamp of the response.\n - `submissionUrl` (str): URL to submit the transcription.\n - `asset` (dict):\n - `id` (int): ID of the associated asset.\n - `status` (str): Current transcription status.\n - `contributors` (int): Number of users who contributed.\n - `undo_available` (bool): Whether rollback is currently possible.\n - `redo_available` (bool): Whether rollforward is currently possible.\n\n Response Format - Error:\n - `error` (str): Explanation of the validation failure.\n - \"It looks like your text contains URLs.\"\n - \"An open transcription already exists\"\n - \"This transcription has been superseded\"\n - \"Invalid supersedes value\"\n\n Example:\n ```json\n {\n \"id\": 125,\n \"sent\": 1716295310.743182,\n \"submissionUrl\": \"/transcriptions/125/submit/\",\n \"text\" : \"Transcription text\\r\\nSecond line\",\n \"asset\": {\n \"id\": 456,\n \"status\": \"in_progress\",\n \"contributors\": 1\n },\n \"undo_available\": true,\n \"redo_available\": false\n }\n ```\n \"\"\"\n asset = get_object_or_404(Asset, pk=asset_pk)\n logger.info(\"Saving transcription for %s (%s)\", asset, asset.id)\n\n if request.user.is_anonymous:\n user = get_anonymous_user()\n else:\n user = request.user\n\n structured_logger.info(\n \"Starting transcription save.\",\n event_code=\"transcription_save_start\",\n user=user,\n asset=asset,\n )\n\n transcription_text = request.POST[\"text\"]\n\n # Check whether this transcription text contains any URLs.\n # If so, ask the user to correct the transcription by removing the URLs.\n url_match = re.search(URL_REGEX, transcription_text)\n if url_match:\n structured_logger.warning(\n \"Transcription save rejected due to URL in text.\",\n event_code=\"transcription_save_rejected\",\n reason=\"Transcription text contains URLs\",\n reason_code=\"url_detected\",\n user=user,\n asset=asset,\n )\n return JsonResponse(\n {\n \"error\": \"It looks like your text contains URLs. \"\n \"Please remove the URLs and try again.\",\n \"error-code\": \"url_detected\",\n },\n status=400,\n )\n\n # Sanitize the text by removing any unacceptable (non-printable) characters.\n # This leverages the shared exporter whitelist and logic so behavior remains\n # consistent across validation and export paths.\n transcription_text = remove_unacceptable_characters(transcription_text)\n\n supersedes_pk = request.POST.get(\"supersedes\")\n superseded = get_transcription_superseded(asset, supersedes_pk)\n if superseded and isinstance(superseded, HttpResponse):\n logger.info(\"Transcription superseded\")\n structured_logger.warning(\n \"Superseded transcription is invalid; aborting save.\",\n event_code=\"transcription_save_aborted\",\n reason=\"Superseded transcription is invalid\",\n reason_code=\"superseded_invalid\",\n user=user,\n asset=asset,\n )\n return superseded\n\n if superseded and (superseded.ocr_generated or superseded.ocr_originated):\n ocr_originated = True\n else:\n ocr_originated = False\n\n transcription = Transcription(\n asset=asset,\n user=user,\n supersedes=superseded,\n text=transcription_text,\n ocr_originated=ocr_originated,\n )\n transcription.full_clean()\n transcription.save()\n logger.info(\"Transction %s saved\", transcription.id)\n structured_logger.info(\n \"Transcription saved successfully.\",\n event_code=\"transcription_save_success\",\n user=user,\n transcription=transcription,\n )\n\n return JsonResponse(\n {\n \"id\": transcription.pk,\n \"sent\": time(),\n \"submissionUrl\": reverse(\"submit-transcription\", args=(transcription.pk,)),\n \"text\": transcription.text,\n \"asset\": {\n \"id\": transcription.asset.id,\n \"status\": transcription.asset.transcription_status,\n \"contributors\": transcription.asset.get_contributor_count(),\n },\n \"undo_available\": transcription.asset.can_rollback()[0],\n \"redo_available\": transcription.asset.can_rollforward()[0],\n },\n status=201,\n )\n\n\n@require_POST\n@validate_anonymous_user\ndef submit_transcription(request: HttpRequest, *, pk: Union[int, str]) -> JsonResponse:\n \"\"\"\n Submit a transcription for review.\n\n Marks the transcription as submitted and clears any rejection state.\n Prevents submission if the transcription has already been accepted or\n superseded.\n\n Anonymous users are supported and handled via `get_anonymous_user()`. The caller\n must be validated via `validate_anonymous_user`.\n\n Args:\n request (HttpRequest): The POST request to submit the transcription.\n pk (int or str): The primary key of the transcription to submit.\n\n Returns:\n response (JsonResponse): A dictionary with the asset status and submission\n metadata, or an error response if submission is not allowed.\n\n Response Format - Success:\n - `id` (int): ID of the submitted transcription.\n - `sent` (float): UNIX timestamp of the response.\n - `asset` (dict):\n - `id` (int): ID of the associated asset.\n - `status` (str): Current transcription status.\n - `contributors` (int): Number of users who contributed.\n - `undo_available` (bool): Always `false` after submission.\n - `redo_available` (bool): Always `false` after submission.\n\n Response Format - Error:\n - `error` (str): Explanation of the submission failure.\n - \"This transcription has already been updated.\"\n\n Example:\n ```json\n {\n \"id\": 126,\n \"sent\": 1716295421.019122,\n \"asset\": {\n \"id\": 456,\n \"status\": \"submitted\",\n \"contributors\": 1\n },\n \"undo_available\": false,\n \"redo_available\": false\n }\n ```\n \"\"\"\n transcription = get_object_or_404(Transcription, pk=pk)\n asset = transcription.asset\n\n logger.info(\n \"Transcription %s submitted for %s (%s)\", transcription.id, asset, asset.id\n )\n\n is_superseded = transcription.asset.transcription_set.filter(supersedes=pk).exists()\n is_already_submitted = transcription.submitted and not transcription.rejected\n\n if is_already_submitted or is_superseded:\n logger.warning(\n (\n \"Submit for review was attempted for invalid transcription \"\n \"record: submitted: %s pk: %d\"\n ),\n str(transcription.submitted),\n pk,\n )\n structured_logger.warning(\n \"Submission rejected: transcription already submitted or superseded.\",\n event_code=\"transcription_submit_rejected\",\n reason=\"Transcription already submitted or superseded\",\n reason_code=\"already_updated\",\n user=request.user,\n transcription=transcription,\n )\n\n return JsonResponse(\n {\n \"error\": \"This transcription has already been updated.\"\n \" Reload the current status before continuing.\"\n },\n status=400,\n )\n\n transcription.submitted = now()\n transcription.rejected = None\n transcription.full_clean()\n transcription.save()\n\n logger.info(\"Transcription %s successfully submitted\", transcription.id)\n structured_logger.info(\n \"Transcription submitted successfully.\",\n event_code=\"transcription_submit_success\",\n user=request.user,\n transcription=transcription,\n )\n\n return JsonResponse(\n {\n \"id\": transcription.pk,\n \"sent\": time(),\n \"asset\": {\n \"id\": transcription.asset.id,\n \"status\": transcription.asset.transcription_status,\n \"contributors\": transcription.asset.get_contributor_count(),\n },\n \"undo_available\": False,\n \"redo_available\": False,\n },\n status=200,\n )\n\n\n@require_POST\n@login_required\n@never_cache\ndef review_transcription(request: HttpRequest, *, pk: Union[int, str]) -> JsonResponse:\n \"\"\"\n Review and accept or reject a submitted transcription.\n\n Only non-authors may accept a transcription. Users are limited by a\n rate limit when accepting transcriptions. Review actions are rejected\n if the transcription has already been reviewed or is invalid.\n\n Args:\n request (HttpRequest): The POST request containing the review action.\n pk (int or str): The primary key of the transcription to review.\n\n Returns:\n response (JsonResponse): A dictionary with updated asset status and\n metadata, or an error response if the review fails.\n\n Response Format - Success:\n - `id` (int): ID of the reviewed transcription.\n - `sent` (float): UNIX timestamp of the response.\n - `asset` (dict):\n - `id` (int): ID of the associated asset.\n - `status` (str): Updated transcription status.\n - `contributors` (int): Number of users who contributed.\n\n Response Format - Error:\n - `error` (str): Explanation of the review failure.\n - \"Invalid action\"\n - \"This transcription has already been reviewed\"\n - \"You cannot accept your own transcription\"\n - Configuration-based rate limit messages\n\n Example:\n ```json\n {\n \"id\": 127,\n \"sent\": 1716295502.642184,\n \"asset\": {\n \"id\": 456,\n \"status\": \"completed\",\n \"contributors\": 2\n }\n }\n ```\n \"\"\"\n action = request.POST.get(\"action\")\n structured_logger.info(\n \"Starting transcription review.\",\n event_code=\"transcription_review_start\",\n user=request.user,\n transcription_id=pk,\n action=action,\n )\n\n if action not in (\"accept\", \"reject\"):\n structured_logger.warning(\n \"Transcription review failed: invalid action.\",\n event_code=\"transcription_review_rejected\",\n reason=\"Invalid review action\",\n reason_code=\"invalid_action\",\n user=request.user,\n transcription_id=pk,\n action=action,\n )\n return JsonResponse({\"error\": \"Invalid action\"}, status=400)\n\n transcription = get_object_or_404(Transcription, pk=pk)\n asset = transcription.asset\n\n logger.info(\n \"Transcription %s reviewed (%s) for %s (%s)\",\n transcription.id,\n action,\n asset,\n asset.id,\n )\n\n if transcription.accepted or transcription.rejected:\n structured_logger.warning(\n \"Review rejected: transcription already reviewed.\",\n event_code=\"transcription_review_rejected\",\n reason=\"Transcription has already been reviewed\",\n reason_code=\"already_reviewed\",\n user=request.user,\n transcription=transcription,\n )\n return JsonResponse(\n {\"error\": \"This transcription has already been reviewed\"}, status=400\n )\n\n if transcription.user.pk == request.user.pk and action == \"accept\":\n logger.warning(\"Attempted self-acceptance for transcription %s\", transcription)\n structured_logger.warning(\n \"Review rejected: user attempted to accept their own transcription.\",\n event_code=\"transcription_review_rejected\",\n reason=\"User attempted to accept their own transcription\",\n reason_code=\"self_accept\",\n user=request.user,\n transcription=transcription,\n )\n return JsonResponse(\n {\"error\": \"You cannot accept your own transcription\"}, status=400\n )\n\n transcription.reviewed_by = request.user\n\n if action == \"accept\":\n concordia_user = ConcordiaUser.objects.get(id=request.user.id)\n try:\n concordia_user.check_and_track_accept_limit(transcription)\n except RateLimitExceededError:\n structured_logger.warning(\n \"Review rejected: user exceeded review rate limit.\",\n event_code=\"transcription_review_rejected\",\n reason=\"User exceeded review rate limit\",\n reason_code=\"rate_limit_exceeded\",\n user=request.user,\n transcription=transcription,\n )\n return JsonResponse(\n {\n \"error\": configuration_value(\"review_rate_limit_banner_message\"),\n \"popupTitle\": configuration_value(\"review_rate_limit_popup_title\"),\n \"popupError\": configuration_value(\n \"review_rate_limit_popup_message\"\n ),\n },\n status=429,\n )\n transcription.accepted = now()\n else:\n transcription.rejected = now()\n\n transcription.full_clean()\n transcription.save()\n\n logger.info(\"Transcription %s successfully reviewed (%s)\", transcription.id, action)\n structured_logger.info(\n \"Transcription review successful.\",\n event_code=\"transcription_review_success\",\n user=request.user,\n transcription=transcription,\n action=action,\n )\n\n return JsonResponse(\n {\n \"id\": transcription.pk,\n \"sent\": time(),\n \"asset\": {\n \"id\": transcription.asset.id,\n \"status\": transcription.asset.transcription_status,\n \"contributors\": transcription.asset.get_contributor_count(),\n },\n },\n status=200,\n )\n\n\n@require_POST\n@login_required\n@atomic\ndef submit_tags(request: HttpRequest, *, asset_pk: Union[int, str]) -> JsonResponse:\n \"\"\"\n Submit a new set of tags for an asset from the current user.\n\n Creates any new tags as needed and updates the user's tag collection\n for the asset. Removes tags that are no longer present in the submission.\n\n Args:\n request (HttpRequest): The POST request containing tag values.\n asset_pk (int or str): The primary key of the asset to tag.\n\n Returns:\n response (JsonResponse): A dictionary containing the updated user-specific\n and global tag lists for the asset.\n\n Response Format - Success:\n - `user_tags` (list[str]): Tags currently assigned to the asset by this user.\n - `all_tags` (list[str]): All tags currently applied to the asset by any user.\n\n Response Format - Error:\n - `error` (list[str]): Validation error messages for malformed/duplicate tags.\n\n Example:\n ```json\n {\n \"user_tags\": [\"map\", \"handwritten\"],\n \"all_tags\": [\"handwritten\", \"map\", \"note\"]\n }\n ```\n \"\"\"\n asset = get_object_or_404(Asset, pk=asset_pk)\n structured_logger.info(\n \"Starting tag submission.\",\n event_code=\"tag_submit_start\",\n user=request.user,\n asset=asset,\n )\n\n user_tags, created = UserAssetTagCollection.objects.get_or_create(\n asset=asset, user=request.user\n )\n\n tags = set(request.POST.getlist(\"tags\"))\n existing_tags = Tag.objects.filter(value__in=tags)\n new_tag_values = tags.difference(i.value for i in existing_tags)\n new_tags = [Tag(value=i) for i in new_tag_values]\n try:\n for i in new_tags:\n i.full_clean()\n except ValidationError as exc:\n structured_logger.warning(\n \"Tag submission rejected: validation error on new tags.\",\n event_code=\"tag_submit_rejected\",\n reason=\"Tag failed validation\",\n reason_code=\"validation_error\",\n user=request.user,\n asset=asset,\n errors=str(exc.messages),\n )\n return JsonResponse({\"error\": exc.messages}, status=400)\n\n Tag.objects.bulk_create(new_tags)\n\n # At this point we now have Tag objects for everything in the POSTed\n # request. We'll add anything which wasn't previously in this user's tag\n # collection and remove anything which is no longer present.\n\n all_submitted_tags = list(existing_tags) + new_tags\n existing_user_tags = user_tags.tags.all()\n\n for tag in all_submitted_tags:\n if tag not in existing_user_tags:\n user_tags.tags.add(tag)\n\n all_tags_qs = Tag.objects.filter(userassettagcollection__asset__pk=asset_pk)\n\n for tag in all_tags_qs:\n if tag not in all_submitted_tags:\n for collection in asset.userassettagcollection_set.all():\n collection.tags.remove(tag)\n\n all_tags = all_tags_qs.order_by(\"value\")\n final_user_tags = user_tags.tags.order_by(\"value\").values_list(\"value\", flat=True)\n all_tags = all_tags.values_list(\"value\", flat=True).distinct()\n\n structured_logger.info(\n \"Tags submitted successfully.\",\n event_code=\"tag_submit_success\",\n user=request.user,\n asset=asset,\n user_tags=[tag.value for tag in user_tags.tags.all()],\n )\n\n return JsonResponse(\n {\"user_tags\": list(final_user_tags), \"all_tags\": list(all_tags)}\n )\n\n\n@ratelimit(\n key=\"header:cf-connecting-ip\", rate=reserve_rate, block=settings.RATELIMIT_BLOCK\n)\n@require_POST\n@never_cache\ndef reserve_asset(request: HttpRequest, *, asset_pk: Union[int, str]) -> JsonResponse:\n \"\"\"\n Attempt to reserve an asset for transcription by the current session.\n\n If no active reservation exists, creates a new one using the session's\n reservation token. If a reservation exists for this session, updates it.\n If the asset is reserved by another session, returns a conflict response.\n Handles reservation release if `release` is set in the request body.\n\n Request Parameters:\n - `release` (bool, optional): If present and true, releases the current\n reservation instead of acquiring or updating it. Example: `\"true\"`\n\n Returns:\n response (JsonResponse or HttpResponse): A dictionary indicating the\n reservation status and token, or an HTTP 408/409 response for timeout\n or conflict.\n\n Response Format - Success:\n - `asset_pk` (int): The ID of the reserved asset.\n - `reservation_token` (str): A unique identifier for the reservation session.\n\n Response Format - Error:\n - `408 Request Timeout`: The current session's reservation is tombstoned.\n - `409 Conflict`: The asset is actively reserved by another session.\n\n Example:\n ```json\n {\n \"asset_pk\": 789,\n \"reservation_token\": \"abc123xyz\"\n }\n ```\n \"\"\"\n\n reservation_token = get_or_create_reservation_token(request)\n structured_logger.info(\n \"Handling reservation request.\",\n event_code=\"asset_reserve_start\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n\n # If the browser is letting us know of a specific reservation release,\n # let it go even if it's within the grace period.\n if request.POST.get(\"release\"):\n with connection.cursor() as cursor:\n cursor.execute(\n \"\"\"\n DELETE FROM concordia_assettranscriptionreservation\n WHERE asset_id = %s and reservation_token = %s\n \"\"\",\n [asset_pk, reservation_token],\n )\n\n # We'll pass the message to the WebSocket listeners before returning it:\n msg = {\"asset_pk\": asset_pk, \"reservation_token\": reservation_token}\n logger.info(\"Releasing reservation with token %s\", reservation_token)\n structured_logger.info(\n \"Releasing asset reservation via client request.\",\n event_code=\"asset_reserve_release\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n reservation_released.send(sender=\"reserve_asset\", **msg)\n return JsonResponse(msg)\n\n # We're relying on the database to meet our integrity requirements and since\n # this is called periodically we want to be fairly fast until we switch to\n # something like Redis.\n\n reservations = AssetTranscriptionReservation.objects.filter(\n asset_id__exact=asset_pk\n )\n\n # Default: pretend there is no activity on the asset\n is_it_already_mine = False\n am_i_tombstoned = False\n is_someone_else_tombstoned = False\n is_someone_else_active = False\n\n if reservations:\n for reservation in reservations:\n if reservation.tombstoned:\n if reservation.reservation_token == reservation_token:\n am_i_tombstoned = True\n logger.debug(\"I'm tombstoned %s\", reservation_token)\n else:\n is_someone_else_tombstoned = True\n logger.debug(\n \"Someone else is tombstoned %s\", reservation.reservation_token\n )\n else:\n if reservation.reservation_token == reservation_token:\n is_it_already_mine = True\n logger.debug(\n \"I already have this active reservation %s\", reservation_token\n )\n if not is_it_already_mine:\n is_someone_else_active = True\n logger.info(\n \"Someone else has this active reservation %s\",\n reservation.reservation_token,\n )\n\n if am_i_tombstoned:\n structured_logger.warning(\n \"Reservation rejected: client is tombstoned.\",\n event_code=\"asset_reserve_rejected\",\n reason=\"Client reservation token is tombstoned\",\n reason_code=\"tombstoned_self\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n return HttpResponse(status=408) # Request Timed Out\n\n if is_someone_else_active:\n structured_logger.warning(\n \"Reservation rejected: asset is reserved by another client.\",\n event_code=\"asset_reserve_rejected\",\n reason=\"Asset is actively reserved by another session\",\n reason_code=\"conflict_active_other\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n return HttpResponse(status=409) # Conflict\n\n if is_it_already_mine:\n # This user already has the reservation and it's not tombstoned\n structured_logger.info(\n \"Reservation updated for client.\",\n event_code=\"asset_reserve_updated\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n msg = update_reservation(asset_pk, reservation_token)\n logger.debug(\"Updating reservation %s\", reservation_token)\n\n if is_someone_else_tombstoned:\n # No reservations = no activity = go ahead and do an insert\n structured_logger.info(\n \"Reservation acquired from tombstoned client.\",\n event_code=\"asset_reserve_from_tombstone\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n msg = obtain_reservation(asset_pk, reservation_token)\n logger.debug(\n \"Obtaining reservation for %s from tombstoned user\", reservation_token\n )\n else:\n # No reservations = no activity = go ahead and do an insert\n structured_logger.info(\n \"Initial reservation acquired (no existing reservations).\",\n event_code=\"asset_reserve_fresh\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n msg = obtain_reservation(asset_pk, reservation_token)\n logger.debug(\"No activity, just get the reservation %s\", reservation_token)\n\n return JsonResponse(msg)\n\n\ndef update_reservation(\n asset_pk: Union[int, str], reservation_token: str\n) -> dict[str, Union[int, str]]:\n \"\"\"\n Update the timestamp on an existing active reservation for an asset.\n\n Refreshes the reservation's `updated_on` field to extend its validity\n and emits the `reservation_obtained` signal.\n\n Args:\n asset_pk (int or str): The primary key of the reserved asset.\n reservation_token (str): The session's reservation token.\n\n Returns:\n response (dict): A dictionary confirming the updated reservation state.\n\n Response Format - Success:\n - `asset_pk` (int): The ID of the reserved asset.\n - `reservation_token` (str): The reservation token used by the session.\n\n Example:\n ```json\n {\n \"asset_pk\": 789,\n \"reservation_token\": \"abc123xyz\"\n }\n ```\n \"\"\"\n structured_logger.info(\n \"Attempting to update reservation timestamp.\",\n event_code=\"reservation_update_start\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n with connection.cursor() as cursor:\n cursor.execute(\n \"\"\"\n UPDATE concordia_assettranscriptionreservation AS atr\n SET updated_on = current_timestamp\n WHERE (\n atr.asset_id = %s\n AND atr.reservation_token = %s\n AND atr.tombstoned != TRUE\n )\n \"\"\".strip(),\n [asset_pk, reservation_token],\n )\n structured_logger.info(\n \"Reservation update SQL executed.\",\n event_code=\"reservation_update_sql_executed\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n # We'll pass the message to the WebSocket listeners before returning it:\n msg = {\"asset_pk\": asset_pk, \"reservation_token\": reservation_token}\n reservation_obtained.send(sender=\"reserve_asset\", **msg)\n structured_logger.info(\n \"Reservation update completed; signal dispatched.\",\n event_code=\"reservation_update_success\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n return msg\n\n\ndef obtain_reservation(\n asset_pk: Union[int, str], reservation_token: str\n) -> dict[str, Union[int, str]]:\n \"\"\"\n Create a new reservation entry for an asset.\n\n Inserts a new reservation row in the database for the given asset and\n session token. Emits the `reservation_obtained` signal to notify listeners.\n\n Args:\n asset_pk (int or str): The primary key of the asset to reserve.\n reservation_token (str): The session's reservation token.\n\n Returns:\n response (dict): A dictionary confirming the newly obtained reservation.\n\n Response Format - Success:\n - `asset_pk` (int): The ID of the reserved asset.\n - `reservation_token` (str): The reservation token used by the session.\n\n Example:\n ```json\n {\n \"asset_pk\": 789,\n \"reservation_token\": \"abc123xyz\"\n }\n ```\n \"\"\"\n structured_logger.info(\n \"Attempting to create new reservation.\",\n event_code=\"reservation_obtain_start\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n with connection.cursor() as cursor:\n cursor.execute(\n \"\"\"\n INSERT INTO concordia_assettranscriptionreservation AS atr\n (asset_id, reservation_token, tombstoned, created_on,\n updated_on)\n VALUES (%s, %s, FALSE, current_timestamp,\n current_timestamp)\n \"\"\".strip(),\n [asset_pk, reservation_token],\n )\n structured_logger.info(\n \"Reservation INSERT executed successfully.\",\n event_code=\"reservation_insert_success\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n # We'll pass the message to the WebSocket listeners before returning it:\n msg = {\"asset_pk\": asset_pk, \"reservation_token\": reservation_token}\n reservation_obtained.send(sender=\"reserve_asset\", **msg)\n structured_logger.info(\n \"Reservation successfully obtained; signal dispatched.\",\n event_code=\"reservation_obtain_success\",\n asset_pk=asset_pk,\n reservation_token=reservation_token,\n )\n return msg\n"
},
{
"path": "concordia/views/assets.py",
"content": "import logging\nimport random\nfrom typing import Any\nfrom urllib.parse import urlencode\n\nfrom django.conf import settings\nfrom django.contrib import messages\nfrom django.contrib.auth.models import User\nfrom django.db.models import QuerySet\nfrom django.db.transaction import atomic\nfrom django.http import Http404, HttpRequest, HttpResponse, HttpResponseRedirect\nfrom django.shortcuts import get_object_or_404, redirect\nfrom django.urls import reverse\nfrom django.utils.decorators import method_decorator\nfrom django.views.decorators.cache import never_cache\nfrom django_ratelimit.decorators import ratelimit\n\nfrom concordia.api_views import APIDetailView\nfrom concordia.forms import TurnstileForm\nfrom concordia.logging import ConcordiaLogger\nfrom concordia.models import (\n Asset,\n AssetTranscriptionReservation,\n Campaign,\n CardFamily,\n Guide,\n Topic,\n TranscriptionStatus,\n TutorialCard,\n UserAssetTagCollection,\n)\nfrom concordia.templatetags.concordia_media_tags import asset_media_url\nfrom concordia.utils import (\n get_anonymous_user,\n get_or_create_reservation_token,\n)\nfrom concordia.utils.next_asset import (\n find_next_reviewable_campaign_asset,\n find_next_reviewable_topic_asset,\n find_next_transcribable_campaign_asset,\n find_next_transcribable_topic_asset,\n find_reviewable_campaign_asset,\n find_transcribable_campaign_asset,\n remove_next_asset_objects,\n)\n\nfrom .decorators import next_asset_rate\nfrom .utils import AnonymousUserValidationCheckMixin\n\nlogger = logging.getLogger(__name__)\nstructured_logger = ConcordiaLogger.get_logger(__name__)\n\n\n@method_decorator(never_cache, name=\"dispatch\")\nclass AssetDetailView(AnonymousUserValidationCheckMixin, APIDetailView):\n \"\"\"\n Display details for a single asset and handle missing assets.\n\n This view handles `GET` and `POST` requests by retrieving the published\n `Asset` that matches the campaign, project and item.\n\n It uses `AnonymousUserValidationCheckMixin` for anonymous-user validation\n and `APIDetailView` for API-driven detail behavior. It overrides\n `dispatch` to log and redirect to the parent campaign page if the asset\n is not found.\n\n Attributes:\n template_name (str): Template used to render the asset detail page.\n \"\"\"\n\n template_name = \"transcriptions/asset_detail.html\"\n\n def dispatch(\n self,\n request: HttpRequest,\n *args: Any,\n **kwargs: Any,\n ) -> HttpResponse:\n try:\n return super().dispatch(request, *args, **kwargs)\n except Http404:\n structured_logger.info(\n \"AssetDetailView: asset not found, redirecting to campaign \" \"page.\",\n event_code=\"asset_detail_not_found_redirect\",\n user=request.user,\n campaign_slug=self.kwargs.get(\"campaign_slug\"),\n project_slug=self.kwargs.get(\"project_slug\"),\n item_id=self.kwargs.get(\"item_id\"),\n asset_slug=self.kwargs.get(\"slug\"),\n )\n campaign = get_object_or_404(\n Campaign.objects.published(), slug=self.kwargs[\"campaign_slug\"]\n )\n return redirect(campaign)\n\n def get_queryset(self) -> QuerySet[Asset]:\n asset_qs = Asset.objects.published().filter(\n item__project__campaign__slug=self.kwargs[\"campaign_slug\"],\n item__project__slug=self.kwargs[\"project_slug\"],\n item__item_id=self.kwargs[\"item_id\"],\n slug=self.kwargs[\"slug\"],\n )\n asset_qs = asset_qs.select_related(\"item__project__campaign\")\n\n return asset_qs\n\n def get_context_data(self, **kwargs: Any) -> dict[str, Any]:\n \"\"\"\n Build the context for the asset detail template.\n\n Constructs a dictionary with the entries described in the context\n format.\n\n Context Format:\n - `asset` (Asset): Asset instance being viewed.\n - `item` (Item): Parent item of the asset.\n - `project` (Project): Parent project of the item.\n - `campaign` (Campaign): Campaign that contains the project.\n - `transcription` (Transcription | None): Latest transcription or\n `None`.\n - `next_open_asset_url` (str): URL to the next transcribable\n asset.\n - `next_review_asset_url` (str): URL to the next reviewable\n asset.\n - `transcription_status` (str): One of the keys from\n `TranscriptionStatus`.\n - `activity_mode` (str): `\"transcribe\"` or `\"review\"`, based on\n the transcription status.\n - `disable_ocr` (bool): Whether OCR should be disabled for this\n asset.\n - `previous_asset_url` (str | None): URL to the previous asset, if\n any.\n - `next_asset_url` (str | None): URL to the next asset, if any.\n - `asset_navigation` (list[tuple[int, str]]): Sequence and slug\n pairs for navigation.\n - `thumbnail_url` (str): URL of the asset thumbnail image.\n - `current_asset_url` (str): Absolute URL of this asset detail\n view.\n - `tags` (list[str]): Sorted tag values applied to the asset.\n - `registered_contributors` (int): Number of users who have\n contributed to the asset.\n - `cards` (list[Card]): Tutorial cards for the campaign or the\n default card set.\n - `guides` (QuerySet[dict[str, Any]] | None): Tutorial guide\n entries.\n - `languages` (list[tuple[str, str]]): Supported language\n code and name pairs.\n - `undo_available` (bool): Whether a rollback is possible.\n - `redo_available` (bool): Whether a rollforward is possible.\n - `turnstile_form` (TurnstileForm): Form for the Turnstile\n widget.\n\n Args:\n **kwargs (Any): Additional keyword arguments passed to the\n superclass implementation.\n\n Returns:\n dict[str, Any]: Context data for rendering the asset detail page.\n \"\"\"\n\n ctx = super().get_context_data(**kwargs)\n asset = ctx[\"asset\"]\n # Bind a new logger so asset and user are always included\n context_logger = structured_logger.bind(user=self.request.user, asset=asset)\n context_logger.info(\n \"AssetDetailView: building context.\",\n event_code=\"asset_detail_context_start\",\n )\n ctx[\"item\"] = item = asset.item\n ctx[\"project\"] = project = item.project\n ctx[\"campaign\"] = project.campaign\n\n transcription = asset.transcription_set.order_by(\"-pk\").first()\n context_logger.debug(\n \"AssetDetailView: latest transcription selected.\",\n event_code=\"asset_detail_latest_transcription\",\n transcription=transcription,\n )\n ctx[\"transcription\"] = transcription\n\n ctx[\"next_open_asset_url\"] = \"%s?%s\" % (\n reverse(\n \"transcriptions:redirect-to-next-transcribable-campaign-asset\",\n kwargs={\"campaign_slug\": project.campaign.slug},\n ),\n urlencode(\n {\"project\": project.slug, \"item\": item.item_id, \"asset\": asset.id}\n ),\n )\n\n ctx[\"next_review_asset_url\"] = \"%s?%s\" % (\n reverse(\n \"transcriptions:redirect-to-next-reviewable-campaign-asset\",\n kwargs={\"campaign_slug\": project.campaign.slug},\n ),\n urlencode(\n {\"project\": project.slug, \"item\": item.item_id, \"asset\": asset.id}\n ),\n )\n\n # We handle the case where an item with no transcriptions should be\n # shown as status=not_started here so the logic does not need to be\n # repeated in templates.\n if transcription:\n for choice_key, choice_value in TranscriptionStatus.CHOICE_MAP.items():\n if choice_value == transcription.status:\n transcription_status = choice_key\n else:\n transcription_status = TranscriptionStatus.NOT_STARTED\n ctx[\"transcription_status\"] = transcription_status\n\n context_logger.debug(\n \"AssetDetailView: computed transcription status.\",\n event_code=\"asset_detail_transcription_status\",\n computed_status=transcription_status,\n asset_status=asset.transcription_status,\n )\n\n if (\n transcription_status == TranscriptionStatus.NOT_STARTED\n or transcription_status == TranscriptionStatus.IN_PROGRESS\n ):\n ctx[\"activity_mode\"] = \"transcribe\"\n ctx[\"disable_ocr\"] = asset.turn_off_ocr()\n else:\n ctx[\"disable_ocr\"] = True\n if transcription_status == TranscriptionStatus.SUBMITTED:\n ctx[\"activity_mode\"] = \"review\"\n\n previous_asset = (\n item.asset_set.published()\n .filter(sequence__lt=asset.sequence)\n .order_by(\"sequence\")\n .last()\n )\n next_asset = (\n item.asset_set.published()\n .filter(sequence__gt=asset.sequence)\n .order_by(\"sequence\")\n .first()\n )\n context_logger.debug(\n \"AssetDetailView: asset navigation resolved.\",\n event_code=\"asset_detail_navigation\",\n previous_asset_id=getattr(previous_asset, \"pk\", None),\n next_asset_id=getattr(next_asset, \"pk\", None),\n )\n if previous_asset:\n ctx[\"previous_asset_url\"] = previous_asset.get_absolute_url()\n if next_asset:\n ctx[\"next_asset_url\"] = next_asset.get_absolute_url()\n\n ctx[\"asset_navigation\"] = (\n item.asset_set.published()\n .order_by(\"sequence\")\n .values_list(\"sequence\", \"slug\")\n )\n\n image_url = asset_media_url(asset)\n if asset.download_url and \"iiif\" in asset.download_url:\n thumbnail_url = asset.download_url.replace(\n \"http://tile.loc.gov\", \"https://tile.loc.gov\"\n )\n thumbnail_url = thumbnail_url.replace(\"/pct:100/\", \"/!512,512/\")\n else:\n thumbnail_url = image_url\n context_logger.debug(\n \"AssetDetailView: thumbnail URL determined.\",\n event_code=\"asset_detail_thumbnail\",\n thumbnail_url=thumbnail_url,\n )\n ctx[\"thumbnail_url\"] = thumbnail_url\n\n ctx[\"current_asset_url\"] = self.request.build_absolute_uri()\n\n tag_groups = UserAssetTagCollection.objects.filter(asset__slug=asset.slug)\n\n tags = set()\n\n for tag_group in tag_groups:\n for tag in tag_group.tags.all():\n tags.add(tag.value)\n\n ctx[\"tags\"] = sorted(tags)\n\n ctx[\"registered_contributors\"] = asset.get_contributor_count()\n\n if project.campaign.card_family:\n card_family = project.campaign.card_family\n else:\n card_family = CardFamily.objects.filter(default=True).first()\n if card_family is not None:\n unordered_cards = TutorialCard.objects.filter(tutorial=card_family)\n ordered_cards = unordered_cards.order_by(\"order\")\n ctx[\"cards\"] = [tutorial_card.card for tutorial_card in ordered_cards]\n\n guides = Guide.objects.order_by(\"order\").values(\"title\", \"body\")\n if guides.count() > 0:\n ctx[\"guides\"] = guides\n\n ctx[\"languages\"] = list(settings.LANGUAGE_CODES.items())\n\n ctx[\"undo_available\"] = asset.can_rollback()[0] if transcription else False\n ctx[\"redo_available\"] = asset.can_rollforward()[0] if transcription else False\n\n ctx[\"turnstile_form\"] = TurnstileForm(auto_id=False)\n\n context_logger.info(\n \"AssetDetailView: context ready.\",\n event_code=\"asset_detail_context_ready\",\n transcription=transcription,\n transcription_status=transcription_status,\n )\n return ctx\n\n\ndef redirect_to_next_asset(\n asset: Asset | None,\n mode: str,\n request: HttpRequest,\n user: User,\n) -> HttpResponseRedirect:\n \"\"\"\n Redirect the user to the appropriate asset view or the homepage.\n\n If an asset is found, this helper creates a reservation for it and\n removes the asset from the relevant caching tables. The user is then\n redirected to the transcription page for that asset.\n\n If no asset is provided, it redirects to the homepage and adds an\n informational message.\n\n Args:\n asset (Asset | None): Asset to redirect to, or `None` if no asset is\n available.\n mode (str): Either `\"transcribe\"` or `\"review\"`, used for messaging.\n request (HttpRequest): Request that initiated the redirect.\n user (User): User being redirected.\n\n Returns:\n HttpResponseRedirect: Redirect to the asset detail page or the\n homepage.\n \"\"\"\n structured_logger.info(\n \"Starting redirect to next asset.\",\n event_code=\"redirect_next_asset_start\",\n user=user,\n mode=mode,\n asset=asset,\n )\n reservation_token = get_or_create_reservation_token(request)\n if asset:\n # We previously created reservations for transcriptions but not\n # reviews. This created a race condition with the next asset caching\n # system because the non-reserved asset could be added into the cache\n # table between when the user was redirected and when they made their\n # own reservation. That could result in the asset being added to the\n # caching system and sent to another user.\n res = AssetTranscriptionReservation(\n asset=asset, reservation_token=reservation_token\n )\n res.full_clean()\n res.save()\n structured_logger.info(\n \"Asset reserved and redirecting to asset detail view.\",\n event_code=\"redirect_next_asset_success\",\n asset=asset,\n user=user,\n )\n remove_next_asset_objects(asset.id)\n return redirect(\n \"transcriptions:asset-detail\",\n asset.item.project.campaign.slug,\n asset.item.project.slug,\n asset.item.item_id,\n asset.slug,\n )\n else:\n no_pages_message = f\"There are no remaining pages to {mode}.\"\n structured_logger.warning(\n \"No available asset to redirect to.\",\n event_code=\"redirect_next_asset_empty\",\n reason=(\"There were no eligible assets found to assign to the user.\"),\n reason_code=\"no_asset_available\",\n asset=asset,\n user=user,\n mode=mode,\n )\n messages.info(request, no_pages_message)\n\n return redirect(\"homepage\")\n\n\n@ratelimit(\n key=\"header:cf-connecting-ip\",\n rate=next_asset_rate,\n group=\"next_asset\",\n block=True,\n)\n@never_cache\n@atomic\ndef redirect_to_next_reviewable_asset(\n request: HttpRequest,\n) -> HttpResponseRedirect:\n \"\"\"\n Redirect the user to a reviewable asset from any active reviewable\n campaign.\n\n This view iterates through campaigns marked as next-reviewable, then\n falls back to other active campaigns if needed. It skips campaigns with\n no eligible assets and uses asset caching when possible.\n\n Args:\n request (HttpRequest): Incoming HTTP request.\n\n Returns:\n HttpResponseRedirect: Redirect to the selected asset or the\n homepage.\n \"\"\"\n structured_logger.info(\n \"Entered redirect_to_next_reviewable_asset view.\",\n event_code=\"redirect_reviewable_entry\",\n user=request.user,\n )\n if not request.user.is_authenticated:\n user = get_anonymous_user()\n else:\n user = request.user\n\n campaign_ids = list(\n Campaign.objects.active()\n .listed()\n .published()\n .get_next_review_campaigns()\n .values_list(\"id\", flat=True)\n )\n structured_logger.debug(\n \"Fetched candidate campaign IDs for reviewable assets.\",\n event_code=\"redirect_reviewable_campaign_ids\",\n user=user,\n campaign_ids=campaign_ids,\n )\n asset = None\n if campaign_ids:\n random.shuffle(campaign_ids) # nosec\n else:\n logger.info(\"No configured reviewable campaigns\")\n structured_logger.info(\n \"No configured reviewable campaigns.\",\n event_code=\"redirect_reviewable_no_campaigns\",\n user=user,\n )\n\n for campaign_id in campaign_ids:\n try:\n campaign = Campaign.objects.get(id=campaign_id)\n except IndexError:\n logger.error(\"Next reviewable campaign %s not found\", campaign_id)\n structured_logger.error(\n \"Failed to retrieve next reviewable campaign by ID.\",\n event_code=\"redirect_reviewable_campaign_missing\",\n reason=(\"Reviewable campaign with specified ID was not found.\"),\n reason_code=\"reviewable_campaign_not_found\",\n user=user,\n campaign_id=campaign_id,\n )\n continue\n asset = find_reviewable_campaign_asset(campaign, user)\n if asset:\n break\n else:\n logger.info(\"No reviewable assets found in %s\", campaign)\n structured_logger.info(\n \"No reviewable assets found in campaign.\",\n event_code=\"redirect_reviewable_campaign_empty_primary\",\n user=user,\n campaign=campaign,\n )\n\n if not asset:\n for campaign in (\n Campaign.objects.active()\n .listed()\n .published()\n .exclude(id__in=campaign_ids)\n .order_by(\"launch_date\")\n ):\n asset = find_reviewable_campaign_asset(campaign, user)\n if asset:\n break\n else:\n logger.info(\"No reviewable assets found in %s\", campaign)\n structured_logger.info(\n \"No reviewable assets found in campaign.\",\n event_code=\"redirect_reviewable_campaign_empty_fallback\",\n user=user,\n campaign=campaign,\n )\n structured_logger.info(\n \"Redirecting to next reviewable asset.\",\n event_code=\"redirect_reviewable_success\",\n user=user,\n asset=asset,\n )\n return redirect_to_next_asset(asset, \"review\", request, user)\n\n\n@ratelimit(\n key=\"header:cf-connecting-ip\",\n rate=next_asset_rate,\n group=\"next_asset\",\n block=True,\n)\n@never_cache\n@atomic\ndef redirect_to_next_transcribable_asset(\n request: HttpRequest,\n) -> HttpResponseRedirect:\n \"\"\"\n Redirect the user to a transcribable asset from any active transcription\n campaign.\n\n This view iterates through campaigns marked as next-transcribable, then\n falls back to other active campaigns if needed. It skips campaigns with\n no eligible assets and uses asset caching when possible.\n\n Args:\n request (HttpRequest): Incoming HTTP request.\n\n Returns:\n HttpResponseRedirect: Redirect to the selected asset or the\n homepage.\n \"\"\"\n structured_logger.info(\n \"Entered redirect_to_next_transcribable_asset view.\",\n event_code=\"redirect_transcribable_entry\",\n user=request.user,\n )\n campaign_ids = list(\n Campaign.objects.active()\n .listed()\n .published()\n .get_next_transcription_campaigns()\n .values_list(\"id\", flat=True)\n )\n structured_logger.debug(\n \"Fetched candidate campaign IDs for transcribable assets.\",\n event_code=\"redirect_transcribable_campaign_ids\",\n user=request.user,\n campaign_ids=campaign_ids,\n )\n asset = None\n if campaign_ids:\n random.shuffle(campaign_ids) # nosec\n else:\n logger.info(\"No configured transcribable campaigns\")\n structured_logger.info(\n \"No configured transcribable campaigns.\",\n event_code=\"redirect_transcribable_no_campaigns\",\n user=request.user,\n )\n\n for campaign_id in campaign_ids:\n try:\n campaign = Campaign.objects.get(id=campaign_id)\n except IndexError:\n logger.error(\"Next transcribable campaign %s not found\", campaign_id)\n structured_logger.error(\n \"Next transcribable campaign ID not found.\",\n event_code=\"redirect_transcribable_campaign_missing\",\n reason=(\"Transcribable campaign with specified ID was not found.\"),\n reason_code=\"transcribable_campaign_not_found\",\n user=request.user,\n campaign_id=campaign_id,\n )\n continue\n asset = find_transcribable_campaign_asset(campaign)\n if asset:\n break\n else:\n logger.info(\"No transcribable assets found in %s\", campaign)\n structured_logger.info(\n \"No transcribable assets found in campaign.\",\n event_code=\"redirect_transcribable_campaign_empty_primary\",\n user=request.user,\n campaign=campaign,\n )\n\n if not asset:\n for campaign in (\n Campaign.objects.active()\n .listed()\n .published()\n .exclude(id__in=campaign_ids)\n .order_by(\"-launch_date\")\n ):\n asset = find_transcribable_campaign_asset(campaign)\n if asset:\n break\n else:\n logger.info(\"No transcribable assets found in %s\", campaign)\n structured_logger.info(\n \"No transcribable assets found in campaign (fallback \" \"loop).\",\n event_code=\"redirect_transcribable_campaign_empty_fallback\",\n user=request.user,\n campaign=campaign,\n )\n\n if not asset:\n logger.info(\"No transcribable assets found in any campaign\")\n structured_logger.info(\n \"No transcribable assets found in any campaign.\",\n event_code=\"redirect_transcribable_no_assets_anywhere\",\n user=request.user,\n )\n\n structured_logger.info(\n \"Redirecting to next transcribable asset.\",\n event_code=\"redirect_transcribable_success\",\n user=request.user,\n asset=asset,\n )\n return redirect_to_next_asset(asset, \"transcribe\", request, request.user)\n\n\n@ratelimit(\n key=\"header:cf-connecting-ip\",\n rate=next_asset_rate,\n group=\"next_asset\",\n block=True,\n)\n@never_cache\n@atomic\ndef redirect_to_next_reviewable_campaign_asset(\n request: HttpRequest,\n *,\n campaign_slug: str,\n) -> HttpResponseRedirect:\n \"\"\"\n Redirect the user to the next reviewable asset within a campaign.\n\n This view redirects within a specific campaign, which may be listed or\n unlisted. It can use optional query parameters to influence which asset\n is prioritized.\n\n Request Parameters:\n project (str): Current project slug.\n item (str): Current item identifier. This is `item_id`, not the item\n primary key.\n asset (int): ID of the most recently reviewed asset.\n\n Args:\n request (HttpRequest): Incoming HTTP request.\n campaign_slug (str): Slug for the target campaign.\n\n Returns:\n HttpResponseRedirect: Redirect to the selected asset or the\n homepage.\n \"\"\"\n structured_logger.info(\n \"Entered redirect_to_next_reviewable_campaign_asset view.\",\n event_code=\"redirect_reviewable_campaign_entry\",\n user=request.user,\n campaign_slug=campaign_slug,\n )\n # Campaign is specified: may be listed or unlisted\n campaign = get_object_or_404(Campaign.objects.published(), slug=campaign_slug)\n project_slug = request.GET.get(\"project\", \"\")\n item_id = request.GET.get(\"item\", \"\")\n asset_pk = request.GET.get(\"asset\", 0)\n structured_logger.debug(\n \"Parsed query parameters for reviewable asset redirection.\",\n event_code=\"redirect_reviewable_campaign_query_params\",\n user=request.user,\n campaign=campaign,\n project_slug=project_slug,\n item_id=item_id,\n asset_pk=asset_pk,\n )\n\n if not request.user.is_authenticated:\n user = get_anonymous_user()\n else:\n user = request.user\n\n # We pass request.user instead of user here to maintain pre-existing\n # behavior (though it is probably unintended).\n # TODO: Re-evaluate whether we should pass in user instead.\n asset = find_next_reviewable_campaign_asset(\n campaign, request.user, project_slug, item_id, asset_pk\n )\n structured_logger.info(\n \"Redirecting to next reviewable asset in campaign.\",\n event_code=\"redirect_reviewable_campaign_success\",\n user=user,\n request_user=request.user,\n asset=asset,\n campaign=campaign, # We log campaign because asset might be None.\n )\n return redirect_to_next_asset(asset, \"review\", request, user)\n\n\n@ratelimit(\n key=\"header:cf-connecting-ip\",\n rate=next_asset_rate,\n group=\"next_asset\",\n block=True,\n)\n@never_cache\n@atomic\ndef redirect_to_next_transcribable_campaign_asset(\n request: HttpRequest,\n *,\n campaign_slug: str,\n) -> HttpResponseRedirect:\n \"\"\"\n Redirect the user to the next transcribable asset within a campaign.\n\n This view redirects within a specific campaign, which may be listed or\n unlisted. It can use optional query parameters to influence which asset\n is prioritized.\n\n Request Parameters:\n project (str): Current project slug.\n item (str): Current item identifier. This is `item_id`, not the item\n primary key.\n asset (int): ID of the most recently transcribed asset.\n\n Args:\n request (HttpRequest): Incoming HTTP request.\n campaign_slug (str): Slug for the target campaign.\n\n Returns:\n HttpResponseRedirect: Redirect to the selected asset or the\n homepage.\n \"\"\"\n structured_logger.info(\n \"Entered redirect_to_next_transcribable_campaign_asset view.\",\n event_code=\"redirect_transcribable_campaign_entry\",\n user=request.user,\n campaign_slug=campaign_slug,\n )\n # Campaign is specified: may be listed or unlisted\n campaign = get_object_or_404(Campaign.objects.published(), slug=campaign_slug)\n project_slug = request.GET.get(\"project\", \"\")\n item_id = request.GET.get(\"item\", \"\")\n asset_pk = request.GET.get(\"asset\", 0)\n structured_logger.debug(\n \"Parsed query parameters for transcribable asset redirection.\",\n event_code=\"redirect_transcribable_campaign_query_params\",\n user=request.user,\n campaign=campaign,\n project_slug=project_slug,\n item_id=item_id,\n asset_pk=asset_pk,\n )\n\n if not request.user.is_authenticated:\n user = get_anonymous_user()\n else:\n user = request.user\n\n asset = find_next_transcribable_campaign_asset(\n campaign, project_slug, item_id, asset_pk\n )\n structured_logger.info(\n \"Redirecting to next transcribable asset in campaign.\",\n event_code=\"redirect_transcribable_campaign_success\",\n user=user,\n asset=asset,\n campaign=campaign, # We log campaign because asset may be None.\n )\n return redirect_to_next_asset(asset, \"transcribe\", request, user)\n\n\n@ratelimit(\n key=\"header:cf-connecting-ip\",\n rate=next_asset_rate,\n group=\"next_asset\",\n block=True,\n)\n@never_cache\n@atomic\ndef redirect_to_next_reviewable_topic_asset(\n request: HttpRequest,\n *,\n topic_slug: str,\n) -> HttpResponseRedirect:\n \"\"\"\n Redirect the user to the next reviewable asset within a topic.\n\n This view redirects within a specific topic, which may be listed or\n unlisted. It can use optional query parameters to influence which asset\n is prioritized.\n\n Request Parameters:\n project (str): Current project slug.\n item (str): Current item identifier. This is `item_id`, not the item\n primary key.\n asset (int): ID of the most recently reviewed asset.\n\n Args:\n request (HttpRequest): Incoming HTTP request.\n topic_slug (str): Slug for the target topic.\n\n Returns:\n HttpResponseRedirect: Redirect to the selected asset or the\n homepage.\n \"\"\"\n structured_logger.info(\n \"Entered redirect_to_next_reviewable_topic_asset view.\",\n event_code=\"redirect_reviewable_topic_entry\",\n user=request.user,\n topic_slug=topic_slug,\n )\n # Topic is specified: may be listed or unlisted\n topic = get_object_or_404(Topic.objects.published(), slug=topic_slug)\n project_slug = request.GET.get(\"project\", \"\")\n item_id = request.GET.get(\"item\", \"\")\n asset_pk = request.GET.get(\"asset\", 0)\n structured_logger.debug(\n \"Parsed query parameters for reviewable topic redirection.\",\n event_code=\"redirect_reviewable_topic_query_params\",\n user=request.user,\n topic=topic,\n project_slug=project_slug,\n item_id=item_id,\n asset_pk=asset_pk,\n )\n\n if not request.user.is_authenticated:\n user = get_anonymous_user()\n else:\n user = request.user\n\n # We pass request.user instead of user here to maintain pre-existing\n # behavior (though it is probably unintended).\n # TODO: Re-evaluate whether we should pass in user instead.\n asset = find_next_reviewable_topic_asset(\n topic, request.user, project_slug, item_id, asset_pk\n )\n structured_logger.info(\n \"Redirecting to next reviewable asset in topic.\",\n event_code=\"redirect_reviewable_topic_success\",\n user=user,\n request_user=request.user,\n asset=asset,\n topic=topic,\n )\n\n return redirect_to_next_asset(asset, \"review\", request, user)\n\n\n@ratelimit(\n key=\"header:cf-connecting-ip\",\n rate=next_asset_rate,\n group=\"next_asset\",\n block=True,\n)\n@never_cache\n@atomic\ndef redirect_to_next_transcribable_topic_asset(\n request: HttpRequest,\n *,\n topic_slug: str,\n) -> HttpResponseRedirect:\n \"\"\"\n Redirect the user to the next transcribable asset within a topic.\n\n This view redirects within a specific topic, which may be listed or\n unlisted. It can use optional query parameters to influence which asset\n is prioritized.\n\n Request Parameters:\n project (str): Current project slug.\n item (str): Current item identifier. This is `item_id`, not the item\n primary key.\n asset (int): ID of the most recently transcribed asset.\n\n Args:\n request (HttpRequest): Incoming HTTP request.\n topic_slug (str): Slug for the target topic.\n\n Returns:\n HttpResponseRedirect: Redirect to the selected asset or the\n homepage.\n \"\"\"\n structured_logger.info(\n \"Entered redirect_to_next_transcribable_topic_asset view.\",\n event_code=\"redirect_transcribable_topic_entry\",\n user=request.user,\n topic_slug=topic_slug,\n )\n # Topic is specified: may be listed or unlisted\n topic = get_object_or_404(Topic.objects.published(), slug=topic_slug)\n project_slug = request.GET.get(\"project\", \"\")\n item_id = request.GET.get(\"item\", \"\")\n asset_pk = request.GET.get(\"asset\", 0)\n structured_logger.debug(\n \"Parsed query parameters for transcribable topic redirection.\",\n event_code=\"redirect_transcribable_topic_query_params\",\n user=request.user,\n topic=topic,\n project_slug=project_slug,\n item_id=item_id,\n asset_pk=asset_pk,\n )\n\n if not request.user.is_authenticated:\n user = get_anonymous_user()\n else:\n user = request.user\n\n asset = find_next_transcribable_topic_asset(topic, project_slug, item_id, asset_pk)\n structured_logger.info(\n \"Redirecting to next transcribable asset in topic.\",\n event_code=\"redirect_transcribable_topic_success\",\n user=user,\n asset=asset,\n topic=topic,\n )\n return redirect_to_next_asset(asset, \"transcribe\", request, user)\n"
},
{
"path": "concordia/views/campaigns.py",
"content": "from typing import Any, Iterable\nfrom urllib.parse import urlencode\n\nfrom django.core.paginator import Paginator\nfrom django.db.models import Count, Q, QuerySet\nfrom django.http import HttpResponse\nfrom django.shortcuts import get_object_or_404, render\nfrom django.utils.decorators import method_decorator\nfrom django.views.generic import TemplateView\n\nfrom concordia.api_views import APIDetailView, APIListView\nfrom concordia.models import (\n STATUS_COUNT_KEYS,\n Asset,\n Campaign,\n Project,\n ResearchCenter,\n SiteReport,\n Topic,\n Transcription,\n TranscriptionStatus,\n)\nfrom concordia.utils.constants import ASSETS_PER_PAGE\n\nfrom .decorators import default_cache_control, user_cache_control\nfrom .utils import (\n annotate_children_with_progress_stats,\n calculate_asset_stats,\n)\n\n\n@method_decorator(default_cache_control, name=\"dispatch\")\nclass CampaignListView(APIListView):\n \"\"\"\n Display a list of active campaigns.\n\n Renders a list of published, listed, and active campaigns ordered by\n their configured ordering and title. Adds context entries for topics\n and completed campaigns for secondary display.\n\n Inherits from APIListView to support both HTML rendering and API\n serialization of campaigns.\n\n Attributes:\n template_name (str): Template used to render the campaign list.\n queryset (QuerySet[Campaign]): The base queryset of campaigns.\n context_object_name (str): The name of the context variable for campaigns.\n\n Returns:\n HttpResponse: Renders the campaign list template with context.\n \"\"\"\n\n template_name = \"transcriptions/campaign_list.html\"\n queryset = (\n Campaign.objects.published()\n .listed()\n .filter(status=Campaign.Status.ACTIVE)\n .order_by(\"ordering\", \"title\")\n )\n context_object_name = \"campaigns\"\n\n def get_context_data(self, **kwargs: Any) -> dict[str, Any]:\n \"\"\"\n Build context data for the campaign list template.\n\n Adds:\n - 'topics': Ordered list of published topics.\n - 'completed_campaigns': Ordered list of completed or retired campaigns.\n\n Args:\n **kwargs: Additional context arguments.\n\n Returns:\n dict[str, Any]: Context data for rendering.\n \"\"\"\n data = super().get_context_data(**kwargs)\n data[\"topics\"] = (\n Topic.objects.published().listed().order_by(\"ordering\", \"title\")\n )\n data[\"completed_campaigns\"] = (\n Campaign.objects.published()\n .listed()\n .filter(status__in=[Campaign.Status.COMPLETED, Campaign.Status.RETIRED])\n .order_by(\"ordering\", \"title\")\n )\n return data\n\n def serialize_context(self, context: dict[str, Any]) -> dict[str, Any]:\n \"\"\"\n Serialize context data for API responses.\n\n Annotates each campaign object with its asset status counts.\n\n Args:\n context (dict[str, Any]): The view context.\n\n Returns:\n dict[str, Any]: Serialized context data for API output.\n \"\"\"\n data = super().serialize_context(context)\n\n object_list = data[\"objects\"]\n\n campaign_stats_qs = (\n Campaign.objects.filter(pk__in=[i[\"id\"] for i in object_list])\n .annotate(\n **{\n v: Count(\n \"project__item__asset\",\n filter=Q(\n project__published=True,\n project__item__published=True,\n project__item__asset__published=True,\n project__item__asset__transcription_status=k,\n ),\n )\n for k, v in STATUS_COUNT_KEYS.items()\n }\n )\n .values(\"pk\", *STATUS_COUNT_KEYS.values())\n )\n\n campaign_asset_counts = {}\n for campaign_stats in campaign_stats_qs:\n campaign_asset_counts[campaign_stats.pop(\"pk\")] = campaign_stats\n\n for obj in object_list:\n obj[\"asset_stats\"] = campaign_asset_counts[obj[\"id\"]]\n\n return data\n\n\n@method_decorator(default_cache_control, name=\"dispatch\")\nclass CompletedCampaignListView(APIListView):\n \"\"\"\n Display a list of completed and/or retired campaigns.\n\n Renders a list of published, listed campaigns filtered by completion or\n retirement status. Optionally filters by research center or campaign type.\n\n Attributes:\n model (Model): The Campaign model class.\n template_name (str): Template used to render the campaign list.\n context_object_name (str): The name of the context variable for campaigns.\n\n Returns:\n HttpResponse: Renders the completed campaign list template with context.\n \"\"\"\n\n model = Campaign\n template_name = \"transcriptions/campaign_list_small_blocks.html\"\n context_object_name = \"campaigns\"\n\n def _get_all_campaigns(self) -> QuerySet[Campaign]:\n \"\"\"\n Retrieve all completed or retired campaigns, optionally filtered by type.\n\n Returns:\n QuerySet[Campaign]: Filtered campaigns.\n \"\"\"\n campaignType = self.request.GET.get(\"type\", None)\n campaigns = Campaign.objects.published().listed()\n if campaignType is None:\n return campaigns.filter(\n status__in=[Campaign.Status.COMPLETED, Campaign.Status.RETIRED]\n )\n elif campaignType == \"retired\":\n status = Campaign.Status.RETIRED\n else:\n status = Campaign.Status.COMPLETED\n\n return campaigns.filter(status=status)\n\n def get_queryset(self) -> QuerySet[Campaign]:\n \"\"\"\n Build the queryset of completed or retired campaigns.\n\n Optionally filters by research center if provided.\n\n Returns:\n QuerySet[Campaign]: The queryset for completed campaigns.\n \"\"\"\n campaigns = self._get_all_campaigns()\n research_center = self.request.GET.get(\"research_center\", None)\n if research_center is not None:\n campaigns = campaigns.filter(research_centers=research_center)\n return campaigns.order_by(\"-completed_date\")\n\n def get_context_data(self, **kwargs: Any) -> dict[str, Any]:\n \"\"\"\n Build context data for the completed campaign list template.\n\n Adds:\n - 'result_count': The number of campaigns in the list.\n - 'research_centers': Distinct research centers for these campaigns.\n\n Args:\n **kwargs: Additional context arguments.\n\n Returns:\n dict[str, Any]: Context data for rendering.\n \"\"\"\n campaigns = self._get_all_campaigns()\n data = super().get_context_data(**kwargs)\n data[\"result_count\"] = self.object_list.count()\n data[\"research_centers\"] = ResearchCenter.objects.filter(\n campaign__in=campaigns\n ).distinct()\n\n return data\n\n\n@method_decorator(default_cache_control, name=\"dispatch\")\nclass CampaignTopicListView(TemplateView):\n \"\"\"\n Display a list of campaigns grouped by topic.\n\n Renders active campaigns, a subset of topics and completed/retired campaigns\n for navigation and discovery pages.\n\n Attributes:\n template_name (str): Template used to render the campaign-topic list page.\n\n Returns:\n HttpResponse: Renders the campaign topic list template with context.\n \"\"\"\n\n template_name = \"transcriptions/campaign_topic_list.html\"\n\n def get(self, request, *args: Any, **kwargs: Any) -> HttpResponse:\n \"\"\"\n Handle GET requests for the campaign-topic list page.\n\n Builds context containing:\n - 'campaigns': Ordered list of active campaigns.\n - 'topics': Ordered list of up to 5 topics.\n - 'completed_campaigns': Ordered list of completed and retired campaigns.\n\n Args:\n request (HttpRequest): The incoming HTTP request.\n *args: Additional positional arguments.\n **kwargs: Additional keyword arguments.\n\n Returns:\n HttpResponse: Rendered campaign topic list page.\n \"\"\"\n data = {}\n data[\"campaigns\"] = (\n Campaign.objects.published()\n .listed()\n .filter(status=Campaign.Status.ACTIVE)\n .annotated()\n .order_by(\"ordering\", \"title\")\n )\n data[\"topics\"] = (\n Topic.objects.published().listed().order_by(\"ordering\", \"title\")[:5]\n )\n data[\"completed_campaigns\"] = (\n Campaign.objects.published()\n .listed()\n .filter(status__in=[Campaign.Status.COMPLETED, Campaign.Status.RETIRED])\n .order_by(\"ordering\", \"title\")\n )\n\n return render(request, self.template_name, data)\n\n\n@method_decorator(default_cache_control, name=\"dispatch\")\nclass CampaignDetailView(APIDetailView):\n \"\"\"\n Display details for a single campaign.\n\n Renders campaign information, associated projects, and aggregated asset\n statistics. Selects different templates based on campaign status\n (active, completed, or retired).\n\n Attributes:\n template_name (str): Template for active campaigns.\n completed_template_name (str): Template for completed campaigns.\n retired_template_name (str): Template for retired campaigns.\n context_object_name (str): Context variable name for the campaign.\n queryset (QuerySet[Campaign]): Base queryset of campaigns.\n\n Returns:\n HttpResponse: Renders the campaign detail template with context.\n \"\"\"\n\n template_name = \"transcriptions/campaign_detail.html\"\n completed_template_name = \"transcriptions/campaign_detail_completed.html\"\n retired_template_name = \"transcriptions/campaign_detail_retired.html\"\n context_object_name = \"campaign\"\n queryset = Campaign.objects.published().order_by(\"title\")\n\n def get_context_data(self, **kwargs: Any) -> dict[str, Any]:\n \"\"\"\n Build context data for the campaign detail page.\n\n Adds:\n - For retired campaigns: contributor and completed counts from SiteReport.\n - For active campaigns: filtered and annotated projects, asset statistics.\n\n Args:\n **kwargs: Additional context arguments.\n\n Returns:\n dict[str, Any]: Context data for rendering.\n \"\"\"\n ctx = super().get_context_data(**kwargs)\n if self.object and self.object.status == Campaign.Status.RETIRED:\n latest_report = SiteReport.objects.filter(campaign=ctx[\"campaign\"]).latest(\n \"created_on\"\n )\n ctx[\"completed_count\"] = latest_report.assets_completed\n ctx[\"contributor_count\"] = latest_report.registered_contributors\n else:\n projects = (\n ctx[\"campaign\"].project_set.published().order_by(\"ordering\", \"title\")\n )\n ctx[\"filters\"] = filters = {}\n filter_by_reviewable = kwargs.get(\"filter_by_reviewable\", False)\n if filter_by_reviewable:\n projects = projects.filter(\n item__asset__transcription__id__in=Transcription.objects.exclude(\n user=self.request.user.id\n ).values_list(\"id\", flat=True)\n )\n ctx[\"filter_assets\"] = True\n projects = projects.annotate(\n **{\n f\"{key}_count\": Count(\n \"item__asset\",\n filter=Q(\n item__published=True,\n item__asset__published=True,\n item__asset__transcription_status=key,\n ),\n )\n for key in TranscriptionStatus.CHOICE_MAP\n }\n )\n\n if filter_by_reviewable:\n status = TranscriptionStatus.SUBMITTED\n else:\n status = self.request.GET.get(\"transcription_status\")\n if status in TranscriptionStatus.CHOICE_MAP:\n projects = projects.exclude(**{f\"{status}_count\": 0})\n # We only want to pass specific QS parameters\n # to lower-level search pages:\n filters[\"transcription_status\"] = status\n ctx[\"sublevel_querystring\"] = urlencode(filters)\n\n annotate_children_with_progress_stats(projects)\n ctx[\"projects\"] = projects\n\n campaign_assets = Asset.objects.filter(\n item__project__campaign=self.object,\n item__project__published=True,\n item__published=True,\n published=True,\n )\n if filter_by_reviewable:\n campaign_assets = campaign_assets.exclude(\n transcription__user=self.request.user.id\n )\n ctx[\"transcription_status\"] = TranscriptionStatus.SUBMITTED\n else:\n ctx[\"transcription_status\"] = status\n\n calculate_asset_stats(campaign_assets, ctx)\n\n return ctx\n\n def serialize_context(self, context: dict[str, Any]) -> dict[str, Any]:\n \"\"\"\n Serialize campaign context data for API responses.\n\n Adds:\n - 'related_links': Helpful Link title and URL pairs for the campaign.\n\n Args:\n context (dict[str, Any]): The view context.\n\n Returns:\n dict[str, Any]: Serialized context data for API output.\n \"\"\"\n ctx = super().serialize_context(context)\n ctx[\"object\"][\"related_links\"] = [\n {\"title\": title, \"url\": url}\n for title, url in self.object.helpfullink_set.values_list(\n \"title\", \"link_url\"\n )\n ]\n return ctx\n\n def get_template_names(self) -> list[str]:\n \"\"\"\n Determine the template to use based on campaign status.\n\n Returns:\n list[str]: List containing the selected template name.\n \"\"\"\n if self.object.status == Campaign.Status.COMPLETED:\n return [self.completed_template_name]\n elif self.object.status == Campaign.Status.RETIRED:\n return [self.retired_template_name]\n return super().get_template_names()\n\n\n@method_decorator(user_cache_control, name=\"dispatch\")\nclass FilteredCampaignDetailView(CampaignDetailView):\n \"\"\"\n Display campaign details with reviewable asset filtering for staff users.\n\n Inherits from CampaignDetailView, overriding context data to include only\n assets eligible for review by staff users when authenticated.\n\n Returns:\n HttpResponse: Renders the filtered campaign detail template with context.\n \"\"\"\n\n def get_context_data(self, **kwargs: Any) -> dict[str, Any]:\n \"\"\"\n Build context data with reviewable asset filtering for staff users.\n\n Adds 'filter_by_reviewable' to kwargs when user is authenticated and staff.\n\n Args:\n **kwargs: Additional context arguments.\n\n Returns:\n dict[str, Any]: Context data for rendering.\n \"\"\"\n if self.request.user.is_authenticated and self.request.user.is_staff:\n kwargs[\"filter_by_reviewable\"] = True\n\n return super().get_context_data(**kwargs)\n\n\n@method_decorator(default_cache_control, name=\"dispatch\")\nclass ReportCampaignView(TemplateView):\n \"\"\"\n Display a report summarizing campaign resources and status.\n\n Renders a paginated report including project-level asset counts, tag counts,\n contributor counts, reviewer counts and transcription status summaries.\n\n Attributes:\n template_name (str): Template used to render the campaign report page.\n\n Returns:\n HttpResponse: Renders the campaign report template with context.\n \"\"\"\n\n template_name = \"transcriptions/campaign_report.html\"\n\n def get(\n self, request, campaign_slug: str, *args: Any, **kwargs: Any\n ) -> HttpResponse:\n \"\"\"\n Handle GET requests for the campaign report page.\n\n Builds context containing:\n - Campaign title and slug\n - Total asset count\n - Paginated projects with asset, tag, transcriber and reviewer counts\n - Transcription status summaries per project\n\n Args:\n request (HttpRequest): The incoming HTTP request.\n campaign_slug (str): Slug for the campaign to report on.\n *args: Additional positional arguments.\n **kwargs: Additional keyword arguments.\n\n Returns:\n HttpResponse: Rendered campaign report page.\n \"\"\"\n campaign = get_object_or_404(Campaign.objects.published(), slug=campaign_slug)\n\n try:\n page = int(self.request.GET.get(\"page\", \"1\"))\n except ValueError:\n page = 1\n\n campaign_assets = Asset.objects.published().filter(\n item__project__campaign=campaign\n )\n\n ctx = {\n \"title\": campaign.title,\n \"campaign_slug\": campaign.slug,\n \"total_asset_count\": campaign_assets.count(),\n }\n\n projects_qs = campaign.project_set.published().order_by(\"title\")\n\n projects_qs = projects_qs.annotate(\n asset_count=Count(\n \"item__asset\",\n filter=Q(item__published=True, item__asset__published=True),\n distinct=True,\n )\n )\n projects_qs = projects_qs.annotate(\n tag_count=Count(\"item__asset__userassettagcollection__tags\", distinct=True)\n )\n projects_qs = projects_qs.annotate(\n transcriber_count=Count(\"item__asset__transcription__user\", distinct=True),\n reviewer_count=Count(\n \"item__asset__transcription__reviewed_by\", distinct=True\n ),\n )\n\n paginator = Paginator(projects_qs, ASSETS_PER_PAGE)\n if page > paginator.num_pages:\n page = 1\n projects_page = paginator.get_page(page)\n\n self.add_transcription_status_summary_to_projects(projects_page)\n\n ctx[\"paginator\"] = paginator\n ctx[\"projects\"] = projects_page\n\n return render(self.request, self.template_name, ctx)\n\n def add_transcription_status_summary_to_projects(\n self, projects: Iterable[Project]\n ) -> None:\n \"\"\"\n Annotate each project with a summary of transcription statuses.\n\n Adds a 'transcription_statuses' attribute to each project, containing\n status names and their respective counts, ordered by status.\n\n Args:\n projects (Iterable): Projects to annotate.\n\n Returns:\n None\n \"\"\"\n status_qs = Asset.objects.filter(\n item__published=True, item__project__in=projects, published=True\n )\n status_qs = status_qs.values_list(\"item__project__id\", \"transcription_status\")\n status_qs = status_qs.annotate(Count(\"transcription_status\"))\n project_statuses = {}\n\n for project_id, status_value, count in status_qs:\n status_name = TranscriptionStatus.CHOICE_MAP[status_value]\n project_statuses.setdefault(project_id, []).append((status_name, count))\n\n # We'll sort the statuses in the same order they're presented in the choices\n # list so the display order will be both stable and consistent with the way\n # we talk about the workflow:\n sort_order = [j for i, j in TranscriptionStatus.CHOICES]\n\n for project in projects:\n statuses = project_statuses.get(project.id, [])\n statuses.sort(key=lambda i: sort_order.index(i[0]))\n project.transcription_statuses = statuses\n"
},
{
"path": "concordia/views/decorators.py",
"content": "from collections.abc import Callable\nfrom functools import wraps\nfrom time import time\n\nfrom django.conf import settings\nfrom django.core.exceptions import ObjectDoesNotExist, ValidationError\nfrom django.http import HttpRequest, JsonResponse\nfrom django.views.decorators.cache import cache_control, never_cache\nfrom django.views.decorators.vary import vary_on_headers\n\nfrom concordia.forms import TurnstileForm\nfrom concordia.logging import ConcordiaLogger\nfrom configuration.utils import configuration_value\nfrom configuration.validation import validate_rate\n\nstructured_logger = ConcordiaLogger.get_logger(__name__)\n\n\ndef default_cache_control(view_function: Callable) -> Callable:\n \"\"\"\n Decorator that applies default cache control headers to public-facing views.\n\n This decorator sets `Cache-Control: public` with a max-age defined in the\n `DEFAULT_PAGE_TTL` Django setting. It also varies the response by the\n `Accept-Encoding` header.\n\n Args:\n view_function (Callable): The view function to decorate.\n\n Returns:\n Callable: The wrapped view function with cache control headers applied.\n \"\"\"\n\n @vary_on_headers(\"Accept-Encoding\")\n @cache_control(public=True, max_age=settings.DEFAULT_PAGE_TTL)\n @wraps(view_function)\n def inner(*args, **kwargs):\n return view_function(*args, **kwargs)\n\n return inner\n\n\ndef user_cache_control(view_function: Callable) -> Callable:\n \"\"\"\n Decorator that applies cache control headers for views varying by session.\n\n This decorator is intended for views that may return different content\n based on whether the user is authenticated. It sets\n `Cache-Control: public` with the `DEFAULT_PAGE_TTL` setting and varies\n the response by both `Accept-Encoding` and `Cookie` headers.\n\n Args:\n view_function (Callable): The view function to decorate.\n\n Returns:\n Callable: The wrapped view function with user-aware cache control\n headers.\n \"\"\"\n\n @vary_on_headers(\"Accept-Encoding\", \"Cookie\")\n @cache_control(public=True, max_age=settings.DEFAULT_PAGE_TTL)\n @wraps(view_function)\n def inner(*args, **kwargs):\n return view_function(*args, **kwargs)\n\n return inner\n\n\ndef validate_anonymous_user(view: Callable) -> Callable:\n \"\"\"\n Decorator that applies anonymous user validation for `POST` requests.\n\n If the user is unauthenticated and submits a `POST` request, this\n decorator checks whether the user has recently passed Turnstile\n validation. If not, it validates the request using a `TurnstileForm`.\n Failing validation returns a 401 JSON response.\n\n The timestamp of a successful validation is stored in the user session\n to avoid re-validating within the configured interval.\n\n Args:\n view (Callable): The view function to wrap.\n\n Returns:\n Callable: The wrapped view function with anonymous user validation\n logic.\n \"\"\"\n\n @wraps(view)\n @never_cache\n def inner(request, *args, **kwargs):\n if not request.user.is_authenticated and request.method == \"POST\":\n # First check if the user has already been validated within the\n # time limit. If so, validation can be skipped.\n turnstile_last_validated = request.session.get(\n \"turnstile_last_validated\", 0\n )\n age = time() - turnstile_last_validated\n if age > settings.ANONYMOUS_USER_VALIDATION_INTERVAL:\n form = TurnstileForm(request.POST)\n if not form.is_valid():\n return JsonResponse(\n {\n \"error\": (\n \"Unable to validate. Please try again or \" \"login.\"\n )\n },\n status=401,\n )\n else:\n # User has been validated, so cache the time in the\n # session.\n request.session[\"turnstile_last_validated\"] = time()\n\n return view(request, *args, **kwargs)\n\n return inner\n\n\ndef reserve_rate(group: str, request: HttpRequest) -> str | None:\n \"\"\"\n Determine the rate limit value for a request.\n\n This helper is used to control throttling behavior. If the user is\n anonymous, it returns a fixed rate limit string, for example \"100/m\".\n Authenticated users are not rate-limited and it returns `None`.\n\n The `group` parameter controls how rate limits are grouped. It defaults\n to the dotted name of the view so each view is treated as its own rate\n limit bucket unless explicitly overridden.\n\n Args:\n group (str): Group name used to bucket rate limits. Defaults to the\n dotted view name if not set manually.\n request (HttpRequest): The incoming HTTP request.\n\n Returns:\n str | None: A rate string such as \"100/m\" for anonymous users, or\n `None` otherwise.\n \"\"\"\n return None if request.user.is_authenticated else \"100/m\"\n\n\ndef next_asset_rate(group: str, request: HttpRequest) -> str | None:\n \"\"\"\n Determine the rate limit value for a next-asset request.\n\n If the user is anonymous, this helper returns a rate limit string from\n the `next_asset_rate_limit` configuration value, for example \"4/m\".\n Authenticated users are not rate-limited and it returns `None`.\n\n The `group` parameter controls how rate limits are grouped. It is used\n internally by `django-ratelimit`. It could be used to return different\n rate limits based on the group, but that is not needed currently.\n\n Args:\n group (str): Group name used to bucket rate limits. Defaults to the\n dotted view name if not set manually.\n request (HttpRequest): The incoming HTTP request.\n\n Returns:\n str | None: A rate string such as \"4/m\" for anonymous users, or\n `None` otherwise.\n \"\"\"\n if request.user.is_authenticated:\n return None\n try:\n rate_limit = configuration_value(\"next_asset_rate_limit\")\n return validate_rate(rate_limit)\n except (ObjectDoesNotExist, ValidationError) as exc:\n structured_logger.warning(\n \"Falling back to default next-asset rate limit.\",\n event_code=\"next_asset_rate_config_fallback\",\n reason=\"Could not load or validate configured rate limit\",\n reason_code=\"config_missing_or_invalid\",\n group=group,\n default_rate=\"4/m\",\n user=request.user,\n error_type=exc.__class__.__name__,\n error=str(exc),\n )\n return \"4/m\"\n"
},
{
"path": "concordia/views/items.py",
"content": "from urllib.parse import urlencode\n\nfrom django.db.models import QuerySet\nfrom django.http import Http404, HttpRequest, HttpResponse\nfrom django.shortcuts import get_object_or_404, redirect\nfrom django.utils.decorators import method_decorator\n\nfrom concordia.api_views import APIListView\nfrom concordia.models import Campaign, Item, TranscriptionStatus\nfrom concordia.utils import get_image_urls_from_asset\n\nfrom .decorators import default_cache_control, user_cache_control\nfrom .utils import calculate_asset_stats\n\n\n@method_decorator(default_cache_control, name=\"dispatch\")\nclass ItemDetailView(APIListView):\n \"\"\"\n Display a paginated list of assets for a specific item.\n\n This view handles GET requests and renders the item detail page,\n which includes the item's assets, context for filtering, and transcription stats.\n\n Uses `APIListView` to support both HTML rendering and optional JSON output for\n frontend consumption.\n\n Attributes:\n template_name (str): Template used to render the asset list.\n context_object_name (str): The variable name for assets in the template.\n paginate_by (int): Number of assets to display per page.\n http_method_names (list[str]): HTTP methods supported by the view.\n \"\"\"\n\n template_name = \"transcriptions/item_detail.html\"\n context_object_name = \"assets\"\n paginate_by = 10\n\n http_method_names = [\"get\", \"options\", \"head\"]\n\n def dispatch(self, request: HttpRequest, *args, **kwargs) -> HttpResponse:\n \"\"\"\n Handle incoming HTTP requests and redirect if the item or campaign is missing.\n\n Args:\n request (HttpRequest): The HTTP request object.\n\n Returns:\n HttpResponse: The response for the view or a redirect to the campaign page.\n \"\"\"\n try:\n return super().dispatch(request, *args, **kwargs)\n except Http404:\n campaign = get_object_or_404(\n Campaign.objects.published(), slug=self.kwargs[\"campaign_slug\"]\n )\n return redirect(campaign)\n\n def _get_assets(self) -> QuerySet:\n \"\"\"\n Retrieve the queryset of published assets for the current item.\n\n If the `filter_by_reviewable` flag is set in `self.kwargs`, excludes any assets\n already transcribed by the current user to allow for review filtering.\n\n Returns:\n QuerySet: The filtered set of published `Asset` objects for the item.\n \"\"\"\n assets = self.item.asset_set.published()\n if self.kwargs.get(\"filter_by_reviewable\", False):\n assets = assets.exclude(transcription__user=self.request.user.id)\n return assets\n\n def get_queryset(self):\n \"\"\"\n Build and return the queryset of assets for the current item.\n\n Retrieves the specified `Item` and filters its published assets based on\n transcription status. If `filter_by_reviewable` is set in `self.kwargs`,\n filters assets not yet transcribed by the current user and restricts the\n status to SUBMITTED.\n\n The resulting queryset is ordered by sequence and annotated for use in the view.\n Also sets `self.filters` for use in context and querystring construction.\n\n Returns:\n QuerySet: The filtered and ordered queryset of `Asset` objects.\n \"\"\"\n self.item = get_object_or_404(\n Item.objects.published().select_related(\"project__campaign\"),\n project__campaign__slug=self.kwargs[\"campaign_slug\"],\n project__slug=self.kwargs[\"project_slug\"],\n item_id=self.kwargs[\"item_id\"],\n )\n\n asset_qs = self._get_assets().order_by(\"sequence\")\n asset_qs = asset_qs.select_related(\n \"item__project__campaign\", \"item__project\", \"item\"\n )\n\n self.filters = {}\n if self.kwargs.get(\"filter_by_reviewable\", False):\n status = TranscriptionStatus.SUBMITTED\n else:\n status = self.request.GET.get(\"transcription_status\")\n if status in TranscriptionStatus.CHOICE_MAP:\n asset_qs = asset_qs.filter(transcription_status=status)\n # We only want to pass specific QS parameters to lower-level search\n # pages so we'll record those here:\n self.filters[\"transcription_status\"] = status\n\n return asset_qs\n\n def get_context_data(self, **kwargs):\n \"\"\"\n Construct the context dictionary for rendering the item detail page.\n\n Adds campaign, project, item and transcription status to the context. Also\n attaches filter state and querystring for pagination or navigation. If review\n filtering is enabled, the context is marked accordingly.\n\n Includes asset-level transcription statistics via `calculate_asset_stats()`.\n\n Returns:\n dict: The context data for the template rendering.\n \"\"\"\n ctx = super().get_context_data(**kwargs)\n\n ctx.update(\n {\n \"campaign\": self.item.project.campaign,\n \"project\": self.item.project,\n \"item\": self.item,\n \"sublevel_querystring\": urlencode(self.filters),\n \"filters\": self.filters,\n }\n )\n\n item_assets = self._get_assets()\n if self.kwargs.get(\"filter_by_reviewable\", False):\n ctx[\"filter_assets\"] = True\n ctx[\"transcription_status\"] = TranscriptionStatus.SUBMITTED\n else:\n ctx[\"transcription_status\"] = self.request.GET.get(\"transcription_status\")\n\n calculate_asset_stats(item_assets, ctx)\n\n return ctx\n\n def serialize_context(self, context: dict) -> dict:\n \"\"\"\n Serialize the context data for JSON responses.\n\n Enhances each serialized asset with its associated image and thumbnail URLs.\n Also includes serialized data for the parent item.\n\n Args:\n context (dict): The original context data returned by `get_context_data()`.\n\n Returns:\n dict: The serialized version of the context suitable for API responses.\n \"\"\"\n data = super().serialize_context(context)\n\n for i, asset in enumerate(context[\"object_list\"]):\n serialized_asset = data[\"objects\"][i]\n image_url, thumbnail_url = get_image_urls_from_asset(asset)\n serialized_asset[\"image_url\"] = image_url\n serialized_asset[\"thumbnail_url\"] = thumbnail_url\n\n data[\"item\"] = self.serialize_object(context[\"item\"])\n return data\n\n\n@method_decorator(user_cache_control, name=\"dispatch\")\nclass FilteredItemDetailView(ItemDetailView):\n \"\"\"\n View that displays only reviewable assets for an item.\n\n Inherits from `ItemDetailView` but overrides queryset and context behavior to\n exclude assets already transcribed by the current user. Used to present assets\n eligible for review.\n \"\"\"\n\n def get_queryset(self):\n \"\"\"\n Modify the queryset to include only reviewable assets.\n\n Sets the `filter_by_reviewable` flag in `self.kwargs` to enable filtering logic\n in the parent view.\n\n Returns:\n QuerySet: A filtered queryset of `Asset` objects for review.\n \"\"\"\n self.kwargs[\"filter_by_reviewable\"] = True\n return super().get_queryset()\n\n def get_context_data(self, **kwargs):\n \"\"\"\n Update the context to reflect that only reviewable assets are being shown.\n\n Ensures `filter_by_reviewable` is set in both `self.kwargs` and `kwargs` so that\n downstream logic (like filtering and labeling) behaves consistently.\n\n Returns:\n dict: The context dictionary for rendering the filtered item detail view.\n \"\"\"\n self.kwargs[\"filter_by_reviewable\"] = True\n kwargs[\"filter_by_reviewable\"] = True\n return super().get_context_data(**kwargs)\n"
},
{
"path": "concordia/views/maintenance_mode.py",
"content": "from time import time\n\nfrom django.core.cache import cache\nfrom django.http import HttpRequest, HttpResponseRedirect\nfrom maintenance_mode.core import set_maintenance_mode\n\n\ndef maintenance_mode_off(request: HttpRequest) -> HttpResponseRedirect:\n \"\"\"\n Deactivates maintenance mode and redirects to the site root.\n\n Only superusers are allowed to use this view. If the requesting user is not a\n superuser, no change is made to the system state.\n\n Returns:\n HttpResponseRedirect: Redirect to the root path with a timestamp parameter\n used for cache busting.\n \"\"\"\n if request.user.is_superuser:\n set_maintenance_mode(False)\n\n # Added cache busting to make sure maintenance mode banner is\n # always displayed/removed\n return HttpResponseRedirect(\"/?t={}\".format(int(time())))\n\n\ndef maintenance_mode_on(request: HttpRequest) -> HttpResponseRedirect:\n \"\"\"\n Activates maintenance mode and redirects to the site root.\n\n Only superusers are allowed to use this view. If the requesting user is not a\n superuser, no change is made to the system state.\n\n Returns:\n HttpResponseRedirect: Redirect to the root path with a timestamp parameter\n used for cache busting.\n \"\"\"\n if request.user.is_superuser:\n set_maintenance_mode(True)\n\n # Added cache busting to make sure maintenance mode banner is\n # always displayed/removed\n return HttpResponseRedirect(\"/?t={}\".format(int(time())))\n\n\ndef maintenance_mode_frontend_available(request: HttpRequest) -> HttpResponseRedirect:\n \"\"\"\n Enables frontend access during maintenance mode and redirects to the site root.\n\n This sets a cache key (`maintenance_mode_frontend_available`) to allow staff and\n superusers to bypass maintenance restrictions while the site is otherwise disabled.\n Only superusers are allowed to use this view.\n\n Returns:\n HttpResponseRedirect: Redirect to the root path with a timestamp parameter\n used for cache busting.\n \"\"\"\n if request.user.is_superuser:\n cache.set(\"maintenance_mode_frontend_available\", True, None)\n\n return HttpResponseRedirect(\"/?t={}\".format(int(time())))\n\n\ndef maintenance_mode_frontend_unavailable(request: HttpRequest) -> HttpResponseRedirect:\n \"\"\"\n Disables frontend access during maintenance mode and redirects to the site root.\n\n This clears the `maintenance_mode_frontend_available` cache key, fully locking out\n all users (including staff) from the site frontend during maintenance mode.\n Only superusers are allowed to use this view.\n\n Returns:\n HttpResponseRedirect: Redirect to the root path with a timestamp parameter\n used for cache busting.\n \"\"\"\n if request.user.is_superuser:\n cache.set(\"maintenance_mode_frontend_available\", False, None)\n\n return HttpResponseRedirect(\"/?t={}\".format(int(time())))\n"
},
{
"path": "concordia/views/projects.py",
"content": "from urllib.parse import urlencode\n\nfrom django.db.models import Count, Q, QuerySet\nfrom django.http import Http404, HttpRequest, HttpResponse\nfrom django.shortcuts import get_object_or_404, redirect\nfrom django.utils.decorators import method_decorator\n\nfrom concordia.api_views import APIListView\nfrom concordia.models import Asset, Campaign, Project, TranscriptionStatus\n\nfrom .decorators import default_cache_control, user_cache_control\nfrom .utils import annotate_children_with_progress_stats, calculate_asset_stats\n\n\n@method_decorator(default_cache_control, name=\"dispatch\")\nclass ProjectDetailView(APIListView):\n \"\"\"\n Display a paginated list of items for a single project.\n\n Handles GET requests for a published project scoped by campaign and project\n slugs. Applies optional filtering to show only items with a specific\n transcription status. Builds context including campaign/project metadata\n and progress statistics.\n\n Attributes:\n template_name (str): Template used for project detail.\n context_object_name (str): Context key under which the list of items is\n exposed to templates.\n paginate_by (int): Number of items per page.\n\n Returns:\n HttpResponse: Rendered project detail page or a redirect if the project\n or campaign cannot be found.\n \"\"\"\n\n template_name = \"transcriptions/project_detail.html\"\n context_object_name = \"items\"\n paginate_by = 10\n\n def dispatch(self, request: HttpRequest, *args, **kwargs) -> HttpResponse:\n \"\"\"\n Dispatch the request or redirect to campaign if the item is missing.\n\n If an `Http404` occurs during normal dispatch (e.g., item not found),\n redirect to the campaign page to keep navigation stable.\n\n Args:\n request (HttpRequest): The incoming request.\n *args: Positional args forwarded to the superclass.\n **kwargs: Keyword args forwarded to the superclass.\n\n Returns:\n HttpResponse: Normal response from `APIListView.dispatch` or a\n redirect to the campaign page when the asset path is invalid.\n \"\"\"\n try:\n return super().dispatch(request, *args, **kwargs)\n except Http404:\n campaign = get_object_or_404(\n Campaign.objects.published(), slug=self.kwargs[\"campaign_slug\"]\n )\n return redirect(campaign)\n\n def get_queryset(self, filter_by_reviewable: bool = False) -> QuerySet:\n \"\"\"\n Return the queryset of items for the current project.\n\n Loads the published project identified by `campaign_slug` and `slug`,\n then builds an ordered queryset of its published items. When\n `filter_by_reviewable` is true, excludes items already transcribed by\n the requesting user. Each item is annotated with per-status counts\n using `TranscriptionStatus.CHOICE_MAP`.\n\n Request Parameters:\n - `transcription_status` (str, optional): If present and valid,\n restrict the queryset to items that have at least one asset in\n that status (items with zero count for that status are excluded).\n\n Args:\n filter_by_reviewable (bool): If true, exclude items containing an\n asset with a transcription by the current user.\n\n Returns:\n QuerySet: Ordered queryset of items annotated with per-status\n counts. Also sets `self.filters` to the propagated filters that\n should appear in sublevel navigation links.\n \"\"\"\n self.project = get_object_or_404(\n Project.objects.published().select_related(\"campaign\"),\n slug=self.kwargs[\"slug\"],\n campaign__slug=self.kwargs[\"campaign_slug\"],\n )\n\n item_qs = self.project.item_set.published().order_by(\"item_id\")\n if filter_by_reviewable:\n item_qs = item_qs.exclude(asset__transcription__user=self.request.user.id)\n item_qs = item_qs.annotate(\n **{\n f\"{key}_count\": Count(\n \"asset\", filter=Q(asset__transcription_status=key)\n )\n for key in TranscriptionStatus.CHOICE_MAP\n }\n )\n\n self.filters: dict[str, str] = {}\n\n if filter_by_reviewable:\n status = TranscriptionStatus.SUBMITTED\n else:\n status = self.request.GET.get(\"transcription_status\")\n if status in TranscriptionStatus.CHOICE_MAP:\n item_qs = item_qs.exclude(**{f\"{status}_count\": 0})\n # We only want to pass specific QS parameters to lower-level search\n # pages so we'll record those here:\n self.filters[\"transcription_status\"] = status\n\n return item_qs\n\n def get_context_data(self, **kws) -> dict[str, object]:\n \"\"\"\n Build context for the project detail template.\n\n Context Format:\n - `items` (QuerySet): Paginated list of project items.\n - `project` (Project): The current project.\n - `campaign` (Campaign): The parent campaign.\n - `filters` (dict[str, str] | absent): Filters applied to this view.\n - `sublevel_querystring` (str | absent): URL-encoded filters to pass\n into sublevel pages.\n - `transcription_status` (str | None): Current status filter or\n derived status when reviewable filtering is active.\n - `filter_assets` (bool | absent): True when items are filtered to\n exclude those transcribed by the current user.\n - `total_assets` (int): Count of assets in the project (published).\n - `completed_assets` (int): Count of completed assets.\n - `in_progress_assets` (int): Count of in-progress assets.\n - `not_started_assets` (int): Count of not-started assets.\n - `submitted_assets` (int): Count of submitted assets.\n - Progress statistics on each item are added in-place by\n `annotate_children_with_progress_stats`.\n\n Args:\n **kws: Optional flags. Recognized:\n - `filter_by_reviewable` (bool): When true, limit assets and\n force `transcription_status` to `SUBMITTED`.\n\n Returns:\n dict[str, object]: Template context including project/campaign\n metadata, filters, and computed statistics.\n \"\"\"\n ctx = super().get_context_data(**kws)\n ctx[\"project\"] = project = self.project\n ctx[\"campaign\"] = project.campaign\n\n if self.filters:\n ctx[\"sublevel_querystring\"] = urlencode(self.filters)\n ctx[\"filters\"] = self.filters\n\n project_assets = Asset.objects.filter(\n item__project=project, published=True, item__published=True\n )\n filter_by_reviewable = kws.get(\"filter_by_reviewable\", False)\n if filter_by_reviewable:\n project_assets = project_assets.exclude(\n transcription__user=self.request.user.id\n )\n ctx[\"filter_assets\"] = True\n ctx[\"transcription_status\"] = TranscriptionStatus.SUBMITTED\n else:\n ctx[\"transcription_status\"] = self.request.GET.get(\"transcription_status\")\n\n calculate_asset_stats(project_assets, ctx)\n\n annotate_children_with_progress_stats(ctx[\"items\"])\n\n return ctx\n\n def serialize_context(self, context: dict[str, object]) -> dict[str, object]:\n \"\"\"\n Serialize context for API responses.\n\n Extends the base list serialization by attaching the serialized project\n object. Mirrors the behavior used elsewhere to pair list payloads with\n their parent container.\n\n Args:\n context (dict[str, object]): The view context to serialize.\n\n Returns:\n dict[str, object]: A JSON-serializable structure including:\n - `results`/pagination fields from the base serializer.\n - `project` (dict): Serialized project metadata.\n \"\"\"\n data = super().serialize_context(context)\n data[\"project\"] = self.serialize_object(context[\"project\"])\n return data\n\n\n@method_decorator(user_cache_control, name=\"dispatch\")\nclass FilteredProjectDetailView(ProjectDetailView):\n \"\"\"\n Project detail view that filters to reviewable items for the user.\n\n This variant restricts the queryset and context to prioritize items that\n are ready for review by the current user (i.e., excludes items with assets\n already transcribed by that user). It also sets the effective status filter\n to `SUBMITTED` in context.\n \"\"\"\n\n def get_queryset(self) -> QuerySet:\n \"\"\"\n Return the review-focused queryset.\n\n Delegates to the parent implementation with `filter_by_reviewable=True`.\n\n Returns:\n QuerySet: Item queryset annotated with status counts and filtered\n for reviewable content.\n \"\"\"\n return super().get_queryset(filter_by_reviewable=True)\n\n def get_context_data(self, **kws) -> dict[str, object]:\n \"\"\"\n Build context with reviewable filtering enabled.\n\n Sets the `filter_by_reviewable` flag before delegating to the parent\n implementation so that downstream context keys (e.g., status and\n `filter_assets`) reflect review-mode behavior.\n\n Args:\n **kws: Context keyword arguments.\n\n Returns:\n dict[str, object]: Context dictionary with reviewable filtering.\n \"\"\"\n kws[\"filter_by_reviewable\"] = True\n\n return super().get_context_data(**kws)\n"
},
{
"path": "concordia/views/rate_limit.py",
"content": "from django.http import HttpRequest, HttpResponse, JsonResponse\nfrom django.shortcuts import render\n\nfrom concordia.utils import request_accepts_json\n\n\ndef ratelimit_view(\n request: HttpRequest, exception: Exception | None = None\n) -> HttpResponse:\n \"\"\"\n Handles requests blocked due to rate limiting (HTTP 429).\n\n Determines whether to return a JSON or HTML response based on the request headers.\n Adds a `Retry-After` header instructing clients to wait 15 minutes before retrying.\n\n Args:\n request (HttpRequest): The incoming request that triggered the rate limit.\n exception (Exception | None): The exception that caused the view to trigger,\n if available.\n\n Returns:\n HttpResponse: A JSON or HTML 429 response with a retry header.\n \"\"\"\n status_code = 429\n\n ctx = {\n \"error\": \"You have been rate-limited. Please try again later.\",\n \"status\": status_code,\n }\n\n if exception is not None:\n ctx[\"exception\"] = str(exception)\n\n if request.headers.get(\n \"x-requested-with\"\n ) == \"XMLHttpRequest\" or request_accepts_json(request):\n response = JsonResponse(ctx, status=status_code)\n else:\n response = render(request, \"429.html\", context=ctx, status=status_code)\n\n response[\"Retry-After\"] = 15 * 60\n\n return response\n"
},
{
"path": "concordia/views/simple_pages.py",
"content": "import datetime\nfrom typing import Any\n\nimport markdown\nfrom django.core.cache import cache\nfrom django.http import HttpRequest, HttpResponse\nfrom django.shortcuts import get_object_or_404, render\nfrom django.template import Context, Template\nfrom django.utils.http import http_date\nfrom django.utils.timezone import now\nfrom django.views.generic import RedirectView\n\nfrom concordia.models import Guide, SimplePage, SiteReport\nfrom concordia.parser import paginate_blog_posts\n\nfrom .decorators import default_cache_control\n\n\n@default_cache_control\ndef simple_page(\n request: HttpRequest,\n path: str | None = None,\n slug: str | None = None,\n body_ctx: dict[str, Any] | None = None,\n template: str = \"static-page.html\",\n) -> HttpResponse:\n \"\"\"\n Renders a simple Markdown-based page stored in the `SimplePage` model.\n\n If no `path` is provided, defaults to the current request path. Markdown is\n rendered with optional associated guide content. Breadcrumbs and language\n detection are computed from the URL structure.\n\n Request Parameters:\n path (str, optional): The database path of the page. Defaults to the\n current request path.\n slug (str, optional): Unused in current logic; passed for route compatibility.\n body_ctx (dict[str, Any], optional): Additional context injected into the page\n body during rendering.\n template (str): Template used to render the page.\n\n Returns:\n HttpResponse: Rendered HTML of the simple page.\n \"\"\"\n if not path:\n path = request.path\n\n if body_ctx is None:\n body_ctx = {}\n\n page = get_object_or_404(SimplePage, path=path)\n\n md = markdown.Markdown(extensions=[\"meta\"])\n\n breadcrumbs = []\n path_components = request.path.strip(\"/\").split(\"/\")\n for i, segment in enumerate(path_components[:-1], start=1):\n breadcrumbs.append(\n (\"/%s/\" % \"/\".join(path_components[0:i]), segment.replace(\"-\", \" \").title())\n )\n breadcrumbs.append((request.path, page.title))\n\n language_code = \"en\"\n if request.path.replace(\"/\", \"\").endswith(\"-esp\"):\n language_code = \"es\"\n\n ctx = {\n \"language_code\": language_code,\n \"title\": page.title,\n \"breadcrumbs\": breadcrumbs,\n }\n\n guide = page.guide_set.all().first()\n if guide is not None:\n html = \"\".join((page.body, guide.body))\n ctx[\"add_navigation\"] = True\n else:\n html = page.body\n if \"add_navigation\" in ctx:\n ctx[\"guides\"] = Guide.objects.order_by(\"order\")\n body = Template(md.convert(html))\n ctx[\"body\"] = body.render(Context(body_ctx))\n ctx.update(body_ctx)\n\n resp = render(request, template, ctx)\n resp[\"Created\"] = http_date(page.created_on.timestamp())\n return resp\n\n\n@default_cache_control\ndef about_simple_page(\n request: HttpRequest, path: str | None = None, slug: str | None = None\n) -> HttpResponse:\n \"\"\"\n Renders the \"about\" simple page with additional cached campaign and blog stats.\n\n Adds the following keys to the context:\n - `report_date` (datetime): Yesterdayβs date.\n - `campaigns_published` (int): Count from active SiteReport.\n - `assets_published` (int): Active + retired total.\n - `assets_completed` (int): Active + retired total.\n - `assets_waiting_review` (int): Active + retired total.\n - `users_activated` (int): From active SiteReport.\n - `blog_posts` (Callable): Reference to blog post fetcher.\n\n Returns:\n HttpResponse: Rendered HTML of the about page with campaign stats.\n \"\"\"\n context_cache_key = \"about_simple_page-about_context\"\n about_context = cache.get(context_cache_key)\n if not about_context:\n try:\n active_campaigns = SiteReport.objects.filter(\n report_name=SiteReport.ReportName.TOTAL\n ).latest()\n except SiteReport.DoesNotExist:\n active_campaigns = SiteReport(\n campaigns_published=0,\n assets_published=0,\n assets_completed=0,\n assets_waiting_review=0,\n users_activated=0,\n )\n try:\n retired_campaigns = SiteReport.objects.filter(\n report_name=SiteReport.ReportName.RETIRED_TOTAL\n ).latest()\n except SiteReport.DoesNotExist:\n retired_campaigns = SiteReport(\n assets_published=0,\n assets_completed=0,\n assets_waiting_review=0,\n )\n about_context = {\n \"report_date\": now() - datetime.timedelta(days=1),\n \"campaigns_published\": active_campaigns.campaigns_published,\n \"assets_published\": active_campaigns.assets_published\n + retired_campaigns.assets_published,\n \"assets_completed\": active_campaigns.assets_completed\n + retired_campaigns.assets_completed,\n \"assets_waiting_review\": active_campaigns.assets_waiting_review\n + retired_campaigns.assets_waiting_review,\n \"users_activated\": active_campaigns.users_activated,\n \"blog_posts\": paginate_blog_posts(),\n \"about_page\": True,\n }\n cache.set(context_cache_key, about_context, 60 * 60)\n\n return simple_page(request, path, slug, about_context)\n\n\n# These views are to make sure various links to help-center URLs don't break\n# when the URLs are changed to not include help-center and can be removed after\n# all links are updated.\n\n\nclass HelpCenterRedirectView(RedirectView):\n def get_redirect_url(self, *args, **kwargs):\n path = kwargs[\"page_slug\"]\n return \"/get-started/\" + path + \"/\"\n\n\nclass HelpCenterSpanishRedirectView(RedirectView):\n def get_redirect_url(self, *args, **kwargs):\n path = kwargs[\"page_slug\"]\n return \"/get-started-esp/\" + path + \"-esp/\"\n\n\n# End of help-center views\n"
},
{
"path": "concordia/views/topics.py",
"content": "from typing import Any\nfrom urllib.parse import urlencode\n\nfrom django.db.models import Count, F, FilteredRelation, Q\nfrom django.utils.decorators import method_decorator\nfrom django.views.decorators.cache import cache_page\n\nfrom concordia.api_views import APIDetailView\nfrom concordia.models import Asset, Topic, TranscriptionStatus\n\nfrom .decorators import default_cache_control\nfrom .utils import annotate_children_with_progress_stats, calculate_asset_stats\n\n\n@method_decorator(default_cache_control, name=\"dispatch\")\n@method_decorator(cache_page(60 * 60, cache=\"view_cache\"), name=\"dispatch\")\nclass TopicDetailView(APIDetailView):\n \"\"\"\n Display a topic and its projects with aggregated progress stats.\n\n Renders the topic detail page with a list of published projects tied to\n the topic, annotated with per-status asset counts. Supports an optional\n transcription-status filter which narrows projects to those containing\n assets in that status and respects per-topic URL filter overrides.\n\n Attributes:\n template_name (str): Template used for topic detail.\n context_object_name (str): Context key for the main object (`topic`).\n queryset (QuerySet[Topic]): Base queryset for lookup and ordering.\n \"\"\"\n\n template_name = \"transcriptions/topic_detail.html\"\n context_object_name = \"topic\"\n queryset = Topic.objects.published().order_by(\"title\")\n\n def get_context_data(self, **kwargs: Any) -> dict[str, object]:\n \"\"\"\n Build context for the topic detail template.\n\n Computes project-level progress annotations and applies an optional\n status filter. Also computes topic-wide asset statistics and prepares\n sublevel querystring parameters for downstream pages.\n\n Request Parameters:\n - `transcription_status` (str, optional): When present and valid,\n filters projects to those that:\n * have at least one asset in the given status, and\n * either have no `pt__url_filter` or have one matching the\n requested status.\n\n Context Format:\n - `topic` (Topic): The topic being viewed.\n - `projects` (QuerySet): Topic projects with:\n * per-status counts (e.g., `submitted_count`), and\n * `topic_ordering` and `topic_url_filter` from the through\n relation.\n - `filters` (dict[str, str]): Applied filter parameters.\n - `sublevel_querystring` (str): URL-encoded `filters` for links.\n - `transcription_status` (str | None): Reflected status filter.\n - Aggregated asset stats for the topic (added by\n `calculate_asset_stats`), including keys such as:\n * `total_assets`, `completed_assets`, `in_progress_assets`,\n `not_started_assets`, `submitted_assets`.\n\n Args:\n **kwargs: Additional context arguments passed by the base class.\n\n Returns:\n dict[str, object]: Context for rendering the topic detail page.\n \"\"\"\n ctx = super().get_context_data(**kwargs)\n topic = ctx[\"topic\"]\n\n status = self.request.GET.get(\"transcription_status\")\n status_valid = status in TranscriptionStatus.CHOICE_MAP\n\n projects = (\n topic.project_set.published().annotate(\n **{\n f\"{key}_count\": Count(\n \"item__asset\",\n filter=Q(\n item__published=True,\n item__asset__published=True,\n item__asset__transcription_status=key,\n ),\n )\n for key in TranscriptionStatus.CHOICE_MAP.keys()\n }\n )\n # Pin the through relation to THIS topic, otherwise it will annotate for\n # each ProjectTopic the project is part of\n .annotate(\n pt=FilteredRelation(\n \"projecttopic\", condition=Q(projecttopic__topic=topic)\n )\n )\n # Pull fields from the pinned alias\n .annotate(\n topic_ordering=F(\"pt__ordering\"),\n topic_url_filter=F(\"pt__url_filter\"),\n )\n )\n\n # If there's a status filter, we want to exclude any projects\n # don't don't have assets in that status, as well as any\n # that have a URL filter that's different than the status filter\n if status_valid:\n ctx[\"transcription_status\"] = status\n projects = projects.filter(\n Q(pt__url_filter__isnull=True)\n | Q(pt__url_filter=\"\")\n | Q(pt__url_filter=status)\n ).exclude(**{f\"{status}_count\": 0})\n\n projects = projects.order_by(\"topic_ordering\", \"campaign__title\", \"title\")\n\n ctx[\"filters\"] = filters = {}\n if status_valid:\n # We only want to pass specific QS parameters to lower-level search pages:\n filters[\"transcription_status\"] = status\n ctx[\"sublevel_querystring\"] = urlencode(filters)\n\n annotate_children_with_progress_stats(projects)\n ctx[\"projects\"] = projects\n\n topic_assets = Asset.objects.filter(\n item__project__topics=self.object,\n item__project__published=True,\n item__published=True,\n published=True,\n )\n\n calculate_asset_stats(topic_assets, ctx)\n\n return ctx\n\n def serialize_context(self, context: dict[str, object]) -> dict[str, object]:\n \"\"\"\n Serialize context for API consumers.\n\n Extends the base serializer with a `related_links` list derived from the\n topic's associated helpful links.\n\n Args:\n context (dict[str, object]): Fully built template context.\n\n Returns:\n dict[str, object]: JSON-serializable payload that includes the base\n fields from `APIDetailView.serialize_context` and:\n - `object.related_links` (list[dict]): Each with:\n * `title` (str)\n * `url` (str)\n \"\"\"\n ctx = super().serialize_context(context)\n ctx[\"object\"][\"related_links\"] = [\n {\"title\": title, \"url\": url}\n for title, url, sequence in self.object.helpfullink_set.values_list(\n \"title\", \"link_url\"\n )\n ]\n return ctx\n"
},
{
"path": "concordia/views/utils.py",
"content": "import datetime\nfrom collections.abc import Iterable\nfrom time import time\n\nfrom django.conf import settings\nfrom django.db.models import Count, Max, Q, QuerySet\nfrom django.db.models.functions import Greatest\nfrom django.http import HttpRequest\nfrom django.utils import timezone\nfrom django.utils.timezone import now\n\nfrom concordia.models import Asset, Transcription, TranscriptionStatus\n\n\ndef _get_pages(request: HttpRequest) -> QuerySet:\n \"\"\"\n Retrieve a filtered and annotated queryset of assets based on user activity.\n\n Filters the Asset queryset by:\n - Activity type (transcribed or reviewed)\n - Transcription status\n - Date range (start, end or both)\n - Campaign ID\n - Last six months of activity\n\n Assets are annotated with:\n - Timestamps of last transcription/review activity\n - Combined latest activity timestamp\n\n Also applies ordering based on the selected sort parameter.\n\n Args:\n request (HttpRequest): The incoming HTTP request with query parameters.\n\n Returns:\n QuerySet: A queryset of `Asset` objects with applied filters and annotations.\n \"\"\"\n user = request.user\n activity = request.GET.get(\"activity\", None)\n\n if activity == \"transcribed\":\n q = Q(transcription__user=user)\n elif activity == \"reviewed\":\n q = Q(transcription__reviewed_by=user)\n else:\n q = Q(transcription__user=user) | Q(transcription__reviewed_by=user)\n assets = Asset.objects.filter(q)\n\n status_list = request.GET.getlist(\"status\")\n if status_list and status_list != []:\n if \"completed\" not in status_list:\n assets = assets.exclude(transcription_status=TranscriptionStatus.COMPLETED)\n if \"submitted\" not in status_list:\n assets = assets.exclude(transcription_status=TranscriptionStatus.SUBMITTED)\n if \"in_progress\" not in status_list:\n assets = assets.exclude(\n transcription_status=TranscriptionStatus.IN_PROGRESS\n )\n\n assets = assets.select_related(\"item\", \"item__project\", \"item__project__campaign\")\n\n assets = assets.annotate(\n last_transcribed=Max(\n \"transcription__created_on\",\n filter=Q(transcription__user=user),\n ),\n last_reviewed=Max(\n \"transcription__updated_on\",\n filter=Q(transcription__reviewed_by=user),\n ),\n latest_activity=Greatest(\n \"last_transcribed\",\n \"last_reviewed\",\n filter=Q(transcription__user=user) | Q(transcription__reviewed_by=user),\n ),\n )\n fmt = \"%Y-%m-%d\"\n start_date = None\n start = request.GET.get(\"start\", None)\n if start is not None and len(start) > 0:\n start_date = timezone.make_aware(datetime.datetime.strptime(start, fmt))\n end_date = None\n end = request.GET.get(\"end\", None)\n if end is not None and len(end) > 0:\n end_date = timezone.make_aware(datetime.datetime.strptime(end, fmt))\n if start_date is not None and end_date is not None:\n end_date += datetime.timedelta(days=1)\n end = end_date.strftime(fmt)\n assets = assets.filter(latest_activity__range=[start, end])\n elif start_date is not None or end_date is not None:\n date = start_date if start_date else end_date\n assets = assets.filter(\n latest_activity__year=date.year,\n latest_activity__month=date.month,\n latest_activity__day=date.day,\n )\n # CONCD-189 only show pages from the last 6 months\n # This should be an aware datetime, not a date. A date is cast\n # to a naive datetime when it's compared to a datetime\n # field, as is being done here\n SIX_MONTHS_AGO = now() - datetime.timedelta(days=6 * 30)\n assets = assets.filter(latest_activity__gte=SIX_MONTHS_AGO)\n order_by = request.GET.get(\"order_by\", \"date-descending\")\n if order_by == \"date-ascending\":\n assets = assets.order_by(\"latest_activity\", \"-id\")\n else:\n assets = assets.order_by(\"-latest_activity\", \"-id\")\n\n campaign_id = request.GET.get(\"campaign\", None)\n if campaign_id is not None:\n assets = assets.filter(item__project__campaign__pk=campaign_id)\n\n return assets\n\n\ndef calculate_asset_stats(asset_qs: QuerySet, ctx: dict) -> None:\n \"\"\"\n Annotates the context dictionary with asset statistics and contributor data.\n\n Computes:\n - Total number of unique contributors across all transcriptions.\n - Count and percentage of assets per transcription status.\n - Labeled status counts for use in progress displays.\n\n Percentages are capped at 99% for values between 99.0 and 99.999... to avoid\n showing 100% prematurely.\n\n Args:\n asset_qs (QuerySet): A queryset of `Asset` objects to calculate statistics on.\n ctx (dict): The context dictionary to populate with computed values.\n\n Returns:\n None\n \"\"\"\n asset_count = asset_qs.count()\n\n trans_qs = Transcription.objects.filter(asset__in=asset_qs).values_list(\n \"user_id\", \"reviewed_by\"\n )\n user_ids = set()\n for i, j in trans_qs.iterator():\n user_ids.add(i)\n user_ids.add(j)\n # Remove null values from the set, if it exists\n try:\n user_ids.remove(None)\n except KeyError:\n pass\n\n ctx[\"contributor_count\"] = len(user_ids)\n\n asset_state_qs = asset_qs.values_list(\"transcription_status\")\n asset_state_qs = asset_state_qs.annotate(Count(\"transcription_status\")).order_by()\n status_counts_by_key = dict(asset_state_qs)\n\n ctx[\"transcription_status_counts\"] = labeled_status_counts = []\n\n for status_key, status_label in TranscriptionStatus.CHOICES:\n value = status_counts_by_key.get(status_key, 0)\n if value:\n pct_raw = 100 * (value / asset_count)\n if pct_raw >= 99 and pct_raw < 100:\n pct = 99\n else:\n pct = round(pct_raw)\n else:\n pct = 0\n\n ctx[f\"{status_key}_percent\"] = pct\n ctx[f\"{status_key}_count\"] = value\n labeled_status_counts.append((status_key, status_label, value))\n\n\ndef annotate_children_with_progress_stats(children: Iterable) -> None:\n \"\"\"\n Annotates child objects with transcription progress statistics.\n\n Each object is expected to have attributes named `{status}_count` corresponding to\n each transcription status key. This function calculates:\n\n - `total_count`: Total asset count for the object.\n - `{status}_percent`: Percentage of total for each transcription status.\n - `lowest_transcription_status`: The first non-zero status in defined order.\n\n Percentages are capped at 99% for values between 99.0 and 99.999... to avoid\n rounding up to 100% prematurely.\n\n Args:\n children (Iterable): A sequence of objects with `{status}_count` attributes.\n\n Returns:\n None\n \"\"\"\n for obj in children:\n counts = {}\n\n for k, __ in TranscriptionStatus.CHOICES:\n counts[k] = getattr(obj, f\"{k}_count\", 0)\n\n obj.total_count = total = sum(counts.values())\n\n lowest_status = None\n\n for k, __ in TranscriptionStatus.CHOICES:\n count = counts[k]\n\n if total > 0:\n pct_raw = 100 * (count / total)\n if pct_raw >= 99 and pct_raw < 100:\n pct = 99\n else:\n pct = round(pct_raw)\n else:\n pct = 0\n\n setattr(obj, f\"{k}_percent\", pct)\n\n if lowest_status is None and count > 0:\n lowest_status = k\n\n obj.lowest_transcription_status = lowest_status\n\n\nclass AnonymousUserValidationCheckMixin:\n \"\"\"\n Mixin that injects anonymous user validation context into class-based views.\n\n Adds a boolean `anonymous_user_validation_required` to the context, indicating\n whether a Turnstile validation prompt should be displayed based on the time since\n the user's last successful validation.\n\n Intended for use with views that already implement `get_context_data()`, such as\n Django's TemplateView or DetailView subclasses.\n \"\"\"\n\n def get_context_data(self, *args, **kwargs) -> dict:\n \"\"\"\n Add anonymous user validation flag to the context.\n\n If the user is unauthenticated and the time since their last validation exceeds\n the configured interval, the flag is set to True. Otherwise, it is set to False.\n\n Returns:\n dict: The updated template context with the validation flag included.\n \"\"\"\n context = super().get_context_data(**kwargs)\n if not self.request.user.is_authenticated:\n turnstile_last_validated = self.request.session.get(\n \"turnstile_last_validated\", 0\n )\n age = time() - turnstile_last_validated\n context[\"anonymous_user_validation_required\"] = (\n age > settings.ANONYMOUS_USER_VALIDATION_INTERVAL\n )\n else:\n context[\"anonymous_user_validation_required\"] = False\n return context\n"
},
{
"path": "concordia/views/visualizations.py",
"content": "from django.core.cache import caches\nfrom django.http import JsonResponse\nfrom django.utils.decorators import method_decorator\nfrom django.views import View\nfrom django.views.decorators.cache import never_cache\n\n\n@method_decorator(never_cache, name=\"dispatch\")\nclass VisualizationDataView(View):\n \"\"\"\n Serve cached visualization data as JSON, returning a 404 JSON error if missing.\n\n A single endpoint that, given a `name` slug in the URL, looks up exactly\n that key in the 'visualization_cache' and returns its contents as JSON.\n If no entry exists under that key, responds with a 404 and a JSON error message.\n\n Attributes:\n cache (BaseCache): The Django cache used to retrieve data.\n\n URL Parameters:\n name (str): The slug identifying which visualization data to return.\n Example: \"daily-transcription-activity-by-campaign\".\n\n Returns:\n JsonResponse:\n - On success: the cached data (any JSON-serializable structure).\n - On failure: a JSON object {\"error\": \"...\"} with HTTP status 404.\n \"\"\"\n\n cache = caches[\"visualization_cache\"]\n\n def get(self, request, name):\n data = self.cache.get(name)\n if data is None:\n return JsonResponse(\n {\"error\": f\"No visualization data found for '{name}'\"}, status=404\n )\n\n return JsonResponse(data)\n"
},
{
"path": "concordia/widgets.py",
"content": "from django import forms\n\n\nclass EmailWidget(forms.EmailInput):\n template_name = \"forms/widgets/email.html\"\n"
},
{
"path": "concordia/wsgi.py",
"content": "\"\"\"\nWSGI config for concordia project.\n\nIt exposes the WSGI callable as a module-level variable named ``application``.\n\nFor more information on this file, see\nhttps://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/\n\"\"\"\n\nfrom django.core.wsgi import get_wsgi_application\n\napplication = get_wsgi_application()\n"
},
{
"path": "configuration/__init__.py",
"content": ""
},
{
"path": "configuration/admin.py",
"content": "from typing import Any\n\nfrom django.contrib import admin, messages\nfrom django.http import HttpRequest, HttpResponse\nfrom django.template.response import TemplateResponse\nfrom django.utils.html import format_html\n\nfrom configuration.models import Configuration\n\n\n@admin.register(Configuration)\nclass ConfigurationAdmin(admin.ModelAdmin):\n \"\"\"\n Admin configuration for the `Configuration` model.\n\n Behavior:\n - Displays the key, raw value, and description in the changelist.\n - Provides a read-only `validated_value` field on the change form that\n shows the interpreted value as returned by `Configuration.get_value`.\n - Overrides `changeform_view` to add a two-step confirmation flow when\n saving changes, including a preview of the parsed value. Also handles\n an explicit cancel action by rebuilding the normal change form\n context rather than delegating to the base implementation.\n \"\"\"\n\n list_display = (\"key\", \"value\", \"description\")\n readonly_fields = (\"validated_value\",)\n\n def validated_value(self, obj: Configuration) -> str:\n \"\"\"\n Render the parsed configuration value and explanatory text.\n\n Notes:\n This method does not alter the base `ModelAdmin` behavior. It is a\n helper used by the change form to display both the interpreted\n value from `Configuration.get_value()` and a short explanation\n that this parsed value is what application code will consume.\n\n Args:\n obj (Configuration): The instance being edited.\n\n Returns:\n str: HTML-safe string produced by `format_html` containing the\n parsed value and explanatory note.\n \"\"\"\n return format_html(\n \"{}
{}
\",\n obj.get_value(),\n \"This is the interpreted value based on the selected data type. \"\n \"This value is what will be seen by the code that uses this \"\n \"configuration.\",\n )\n\n def changeform_view(\n self,\n request: HttpRequest,\n object_id: str | None = None,\n form_url: str = \"\",\n extra_context: dict[str, Any] | None = None,\n ) -> HttpResponse:\n \"\"\"\n Override the base change form view to add a confirmation step.\n\n Differences from the base implementation:\n - On initial POST, validate the form and, if valid, render a\n confirmation template that previews the parsed value produced by\n `Configuration.get_value()`.\n - On confirmation POST (`_confirm_update`), save the instance and\n show a success message.\n - On cancel POST (`cancel_update`), rebuild the standard change\n form context manually and re-render the change form instead of\n delegating to the base method (which would otherwise proceed with\n the change because it is a POST).\n - For all other flows, fall back to the base implementation.\n\n Args:\n request (HttpRequest): The current request.\n object_id (str | None): Primary key of the object being edited.\n form_url (str): Form action URL.\n extra_context (dict[str, Any] | None): Extra template context.\n\n Returns:\n HttpResponse: Either the confirmation screen, the re-rendered\n change form, or the default response from the base view.\n \"\"\"\n obj = self.get_object(request, object_id)\n\n if request.method == \"POST\":\n if \"_confirm_update\" in request.POST:\n # Second POST: confirmation of update\n form = self.get_form(request, obj)(request.POST, instance=obj)\n if form.is_valid():\n form.save()\n self.message_user(request, \"Configuration updated and cached.\")\n return self.response_post_save_change(request, form.instance)\n else:\n self.message_user(\n request, \"Invalid data on confirmation.\", level=messages.ERROR\n )\n elif \"cancel_update\" in request.POST:\n form = self.get_form(request, obj)(request.POST, instance=obj)\n\n admin_form = admin.helpers.AdminForm(\n form,\n list(self.get_fieldsets(request, obj)),\n self.get_prepopulated_fields(request, obj),\n self.get_readonly_fields(request, obj),\n model_admin=self,\n )\n # We unfortunately have to manually construct this context, since using\n # super causes it to just perform the cancelled change, because this is\n # a POST request\n context = {\n **self.admin_site.each_context(request),\n \"title\": f\"Edit Configuration: {obj.key}\",\n \"adminform\": admin_form,\n \"inline_admin_formsets\": [],\n \"media\": self.media + form.media,\n \"object_id\": object_id,\n \"original\": obj,\n \"opts\": self.model._meta,\n \"add\": False,\n \"change\": True,\n \"is_popup\": False,\n \"save_as\": self.save_as,\n \"has_view_permission\": self.has_view_permission(request, obj),\n \"has_add_permission\": self.has_add_permission(request),\n \"has_change_permission\": self.has_change_permission(request, obj),\n \"has_delete_permission\": self.has_delete_permission(request, obj),\n \"form_url\": form_url,\n \"to_field\": None,\n \"has_editable_inline_admin_formsets\": False,\n }\n return self.render_change_form(\n request,\n context=context,\n add=False,\n change=True,\n form_url=form_url,\n obj=obj,\n )\n\n else:\n # First POST: validate and show confirmation screen\n form = self.get_form(request, obj)(request.POST, instance=obj)\n if form.is_valid():\n new_instance = form.save(commit=False)\n try:\n parsed_value = new_instance.get_value()\n except Exception as e:\n self.message_user(\n request, f\"Validation failed: {e}\", level=messages.ERROR\n )\n return super().changeform_view(\n request, object_id, form_url, extra_context=extra_context\n )\n\n context = {\n \"title\": (\n f\"Confirm Update of Configuration '{new_instance.key}'\"\n ),\n \"original\": self.model._default_manager.get(pk=obj.pk),\n \"new_instance\": new_instance,\n \"parsed_value\": parsed_value,\n \"opts\": self.model._meta,\n \"object_id\": object_id,\n \"form_url\": form_url,\n \"request\": request,\n }\n return TemplateResponse(\n request, \"admin/configuration_confirm_update.html\", context\n )\n\n return super().changeform_view(request, object_id, form_url, extra_context)\n"
},
{
"path": "configuration/apps.py",
"content": "from django.apps import AppConfig\n\n\nclass ConfigurationConfig(AppConfig):\n default_auto_field = \"django.db.models.BigAutoField\"\n name = \"configuration\"\n\n def ready(self):\n import configuration.signals # NOQA\n"
},
{
"path": "configuration/management/__init__.py",
"content": ""
},
{
"path": "configuration/management/commands/__init__.py",
"content": ""
},
{
"path": "configuration/management/commands/configcache.py",
"content": "from django.core.cache import caches\nfrom django.core.management.base import BaseCommand\n\n\nclass Command(BaseCommand):\n help = \"Fetch a value from the configuration cache by key.\" # NOQA: A003\n\n def add_arguments(self, parser):\n parser.add_argument(\"key\", type=str, help=\"The cache key to retrieve\")\n\n def handle(self, *args, **options):\n config_cache = caches[\"configuration_cache\"]\n key = options[\"key\"]\n cache_key = f\"config_{key}\"\n value = config_cache.get(cache_key)\n\n if value is None:\n self.stdout.write(self.style.WARNING(f\"Key '{key}' not found in cache.\"))\n else:\n self.stdout.write(self.style.SUCCESS(f\"Key '{key}' found:\"))\n self.stdout.write(str(value))\n"
},
{
"path": "configuration/migrations/0001_initial.py",
"content": "# Generated by Django 4.2.16 on 2025-02-25 19:21\n\nfrom django.db import migrations, models\n\n\nclass Migration(migrations.Migration):\n\n initial = True\n\n dependencies = []\n\n operations = [\n migrations.CreateModel(\n name=\"Configuration\",\n fields=[\n (\n \"id\",\n models.BigAutoField(\n auto_created=True,\n primary_key=True,\n serialize=False,\n verbose_name=\"ID\",\n ),\n ),\n (\n \"key\",\n models.CharField(\n help_text=\"Unique identifier for the configuration setting\",\n max_length=255,\n unique=True,\n ),\n ),\n (\n \"value\",\n models.TextField(help_text=\"Value of the configuration setting\"),\n ),\n (\n \"data_type\",\n models.CharField(\n choices=[\n (\"text\", \"Plain text\"),\n (\"number\", \"Number\"),\n (\"boolean\", \"Boolean\"),\n (\"json\", \"JSON\"),\n (\"html\", \"HTML\"),\n ],\n default=\"text\",\n help_text=\"Data type of the value\",\n max_length=10,\n ),\n ),\n (\n \"description\",\n models.TextField(\n blank=True,\n help_text=\"Optional description of the configuration setting\",\n ),\n ),\n ],\n ),\n ]\n"
},
{
"path": "configuration/migrations/0002_populate_configurations.py",
"content": "# Generated by Django 4.2.16 on 2025-02-25 20:15\n\nfrom django.db import migrations\n\n\ndef populate_configuration(apps, schema_editor):\n Configuration = apps.get_model(\"configuration\", \"Configuration\")\n\n # List of initial configuration entries\n initial_data = [\n {\n \"key\": \"review_rate_limit_popup_message\",\n \"data_type\": \"html\",\n \"value\": \"Volunteers can only accept {% configuration_value 'review_rate_limit' %} pages per minute.
\\r\\nPlease read all transcriptions completely to ensure they are whole and accurate. See review instructions.
\",\n \"description\": \"Message shown in the pop-up when a user exceeds the review rate limit\",\n },\n {\n \"key\": \"review_rate_limit_popup_title\",\n \"data_type\": \"html\",\n \"value\": \"You cannot yet accept this page\",\n \"description\": \"Title of the error pop-up displayed when a user exceeds the review rate limit\",\n },\n {\n \"key\": \"review_rate_limit\",\n \"data_type\": \"number\",\n \"value\": \"4\",\n \"description\": \"Number of reviews allowed per minute\",\n },\n {\n \"key\": \"review_rate_limit_banner_message\",\n \"data_type\": \"html\",\n \"value\": \"You cannot yet accept this page. Volunteers can only accept {% configuration_value 'review_rate_limit' %} pages per minute. See review instructions .\",\n \"description\": \"Message to display on the banner when a user exceeds the review rate limit\",\n },\n ]\n\n # Insert data into the database\n for entry in initial_data:\n Configuration.objects.update_or_create(key=entry[\"key\"], defaults=entry)\n\n\ndef revert_populate_configuration(apps, schema_editor):\n # We can't actually revert the data to the state it was before,\n # and there's no actual need to, but we need this function to be\n # able to reverse this migration\n pass\n\n\nclass Migration(migrations.Migration):\n\n dependencies = [\n (\"configuration\", \"0001_initial\"),\n ]\n\n operations = [\n migrations.RunPython(populate_configuration, revert_populate_configuration),\n ]\n"
},
{
"path": "configuration/migrations/0003_populate_retry_configurations.py",
"content": "# Generated by Django 4.2.16 on 2025-03-03 20:51\n\nfrom django.db import migrations\n\n\ndef populate_retry_configuration(apps, schema_editor):\n Configuration = apps.get_model(\"configuration\", \"Configuration\")\n\n # List of initial retry configuration entries\n initial_data = [\n {\n \"key\": \"asset_image_import_max_retries\",\n \"data_type\": \"number\",\n \"value\": \"3\",\n \"description\": \"The maximum number of times to retry downloading \"\n \"an asset image during import\",\n },\n {\n \"key\": \"asset_image_import_max_retry_delay\",\n \"data_type\": \"number\",\n \"value\": \"10\",\n \"description\": \"The number of minutes to wait before retrying downloading \"\n \"an image during import. A non-positive number will disable retries. Should \"\n \"not be greater than around 45 to avoid the system duplicating the task.\",\n },\n ]\n\n # Insert data into the database\n for entry in initial_data:\n Configuration.objects.update_or_create(key=entry[\"key\"], defaults=entry)\n\n\ndef revert_populate_retry_configuration(apps, schema_editor):\n # We can't actually revert the data to the state it was before,\n # and there's no actual need to, but we need this function to be\n # able to reverse this migration\n pass\n\n\nclass Migration(migrations.Migration):\n\n dependencies = [\n (\"configuration\", \"0002_populate_configurations\"),\n ]\n\n operations = [\n migrations.RunPython(\n populate_retry_configuration, revert_populate_retry_configuration\n ),\n ]\n"
},
{
"path": "configuration/migrations/0004_alter_configuration_options.py",
"content": "# Generated by Django 4.2.16 on 2025-03-18 20:01\n\nfrom django.db import migrations\n\n\nclass Migration(migrations.Migration):\n\n dependencies = [\n (\"configuration\", \"0003_populate_retry_configurations\"),\n ]\n\n operations = [\n migrations.AlterModelOptions(\n name=\"configuration\",\n options={\"ordering\": [\"key\"]},\n ),\n ]\n"
},
{
"path": "configuration/migrations/0005_alter_configuration_data_type.py",
"content": "# Generated by Django 4.2.22 on 2025-07-29 17:28\n\nfrom django.db import migrations, models\n\n\nclass Migration(migrations.Migration):\n\n dependencies = [\n (\"configuration\", \"0004_alter_configuration_options\"),\n ]\n\n operations = [\n migrations.AlterField(\n model_name=\"configuration\",\n name=\"data_type\",\n field=models.CharField(\n choices=[\n (\"text\", \"Plain text\"),\n (\"number\", \"Number\"),\n (\"boolean\", \"Boolean\"),\n (\"json\", \"JSON\"),\n (\"html\", \"HTML\"),\n (\"rate\", \"Rate\"),\n ],\n default=\"text\",\n help_text=\"Data type of the value\",\n max_length=10,\n ),\n ),\n ]\n"
},
{
"path": "configuration/migrations/0006_populate_next_asset_rate_limit.py",
"content": "from django.db import migrations\n\n\ndef populate_configuration(apps, schema_editor):\n Configuration = apps.get_model(\"configuration\", \"Configuration\")\n\n initial_data = [\n {\n \"key\": \"next_asset_rate_limit\",\n \"data_type\": \"rate\",\n \"value\": \"4/m\",\n \"description\": \"Rate limit of anonymous users for the next_*_asset views. Format is 'X/u', where 'X' is the number of requests and 'u' is 's', 'm', 'h' or'd' (second, minute, hour or day). '5/s' means 'five per second'.\",\n },\n ]\n\n for entry in initial_data:\n Configuration.objects.update_or_create(key=entry[\"key\"], defaults=entry)\n\n\ndef revert_populate_configuration(apps, schema_editor):\n # We can't actually revert the data to the state it was before,\n # and there's no actual need to, but we need this function to be\n # able to reverse this migration\n pass\n\n\nclass Migration(migrations.Migration):\n\n dependencies = [\n (\"configuration\", \"0005_alter_configuration_data_type\"),\n ]\n\n operations = [\n migrations.RunPython(populate_configuration, revert_populate_configuration),\n ]\n"
},
{
"path": "configuration/migrations/__init__.py",
"content": ""
},
{
"path": "configuration/models.py",
"content": "import json\nfrom typing import Any\n\nfrom django import template\nfrom django.core.exceptions import ValidationError\nfrom django.db import models\n\nfrom configuration.validation import validate_rate\n\n\nclass Configuration(models.Model):\n \"\"\"\n Key/value configuration model with typed decoding.\n\n Purpose:\n Store site configuration as string values and expose a helper that\n converts the stored text into a concrete Python type based on\n `data_type`.\n\n Fields:\n key (models.CharField): Unique identifier for the setting.\n data_type (models.CharField): One of `DataType` choices indicating how\n `value` should be interpreted.\n value (models.TextField): Raw text representation of the value.\n description (models.TextField): Optional human-readable description.\n\n Meta:\n ordering: Sorted by `key`.\n \"\"\"\n\n class DataType(models.TextChoices):\n \"\"\"\n Supported data types for decoding `value` in `get_value`.\n \"\"\"\n\n TEXT = \"text\", \"Plain text\"\n NUMBER = \"number\", \"Number\"\n BOOLEAN = \"boolean\", \"Boolean\"\n JSON = \"json\", \"JSON\"\n HTML = \"html\", \"HTML\"\n RATE = \"rate\", \"Rate\"\n\n key = models.CharField(\n max_length=255,\n unique=True,\n help_text=\"Unique identifier for the configuration setting\",\n )\n data_type = models.CharField(\n max_length=10,\n choices=DataType.choices,\n default=DataType.TEXT,\n help_text=\"Data type of the value\",\n )\n value = models.TextField(help_text=\"Value of the configuration setting\")\n description = models.TextField(\n blank=True, help_text=\"Optional description of the configuration setting\"\n )\n\n class Meta:\n ordering = [\"key\"]\n\n def __str__(self) -> str:\n \"\"\"\n Return the configuration key for display.\n \"\"\"\n return self.key\n\n def get_value(self) -> \"Any\":\n \"\"\"\n Decode and return `value` according to `data_type`.\n\n Behavior:\n - `NUMBER`: Try `int(value)`, else try `float(value)`, else return 0.\n - `BOOLEAN`: Return True if `value.lower() == \"true\"`, else False.\n - `JSON`: Parse with `json.loads(value)` and return the result.\n - `HTML`: Render `value` through Django's template engine with an\n empty context and return the rendered string.\n - `RATE`: Validate using `validate_rate(value)`. If validation\n fails, return an empty string. Otherwise return the validated\n value as provided by `validate_rate`.\n - `TEXT` or any unrecognized type: Return `value` unchanged.\n\n Returns:\n Any: Decoded value. The concrete type depends on `data_type` and\n may be `int`, `float`, `bool`, `str`, `dict`, `list`, or a value\n returned by `validate_rate`.\n\n Raises:\n json.JSONDecodeError: If `data_type` is `JSON` and `value` is not\n valid JSON.\n \"\"\"\n if self.data_type == Configuration.DataType.NUMBER:\n try:\n return int(self.value)\n except ValueError:\n try:\n return float(self.value)\n except ValueError:\n return 0\n elif self.data_type == Configuration.DataType.BOOLEAN:\n if self.value.lower() == \"true\":\n return True\n else:\n return False\n elif self.data_type == Configuration.DataType.JSON:\n return json.loads(self.value)\n elif self.data_type == Configuration.DataType.HTML:\n value = template.Template(self.value)\n return value.render(template.Context({}))\n elif self.data_type == Configuration.DataType.RATE:\n try:\n return validate_rate(self.value)\n except ValidationError:\n return \"\"\n else:\n # DataType.TEXT or an unknown type,\n # so just return the value itself\n return self.value\n"
},
{
"path": "configuration/signals.py",
"content": "from django.db.models.signals import post_save\nfrom django.dispatch import receiver\n\nfrom configuration.models import Configuration\nfrom configuration.utils import cache_configuration_value\n\n\n@receiver(post_save, sender=Configuration)\ndef update_cached_configuration_value(\n sender: type[Configuration], *, instance: Configuration, **kwargs\n) -> None:\n \"\"\"\n Post-save signal handler that updates the cached configuration value.\n\n Behavior:\n - Parse the instance value using `Configuration.get_value()`.\n - If parsing succeeds, write the parsed value to the cache via\n `cache_configuration_value`.\n - If parsing raises any exception, skip caching to avoid persisting an\n invalid value.\n\n Signals:\n Connected to `django.db.models.signals.post_save` for\n `configuration.models.Configuration`.\n\n Args:\n sender (type[Configuration]): The model class that sent the signal.\n instance (Configuration): The saved instance whose parsed value should\n be cached.\n\n Returns:\n None\n \"\"\"\n try:\n value = instance.get_value()\n except Exception:\n # Do not cache if value is invalid\n return\n cache_configuration_value(instance.key, value)\n"
},
{
"path": "configuration/templates/admin/configuration_confirm_update.html",
"content": "{% extends \"admin/base_site.html\" %}\n{% load i18n %}\n\n{% block content %}\n \n\n \n{% endblock %}\n"
},
{
"path": "configuration/templatetags/__init__.py",
"content": ""
},
{
"path": "configuration/templatetags/configuration_tags.py",
"content": "from typing import Any\n\nfrom django import template\n\nfrom configuration.models import Configuration\nfrom configuration.utils import configuration_value as _configuration_value\n\nregister = template.Library()\n\n\n@register.simple_tag\ndef configuration_value(key: str) -> Any:\n \"\"\"\n Return the parsed configuration value for a key, for use in templates.\n\n Behavior:\n Delegates to `configuration.utils.configuration_value` to fetch and parse\n the value (including any casting based on the configured data type).\n If the configuration is missing or parsing raises an exception of any\n kind, return an empty string to keep template rendering resilient.\n\n Args:\n key (str): The unique configuration key.\n\n Returns:\n Any: The parsed value when available and valid; otherwise an empty string.\n \"\"\"\n try:\n return _configuration_value(key)\n except (Configuration.DoesNotExist, Exception):\n # Return an empty string if the key does not exist or parsing fails\n return \"\"\n"
},
{
"path": "configuration/tests/__init__.py",
"content": ""
},
{
"path": "configuration/tests/test_admin.py",
"content": "from django.contrib.auth import get_user_model\nfrom django.test import TestCase\nfrom django.urls import reverse\n\nfrom configuration.models import Configuration\n\n\nclass TestConfigurationAdmin(TestCase):\n def setUp(self):\n self.superuser = get_user_model().objects.create_superuser(\n username=\"admin\",\n email=\"admin@example.com\",\n password=\"adminpass\", # nosec\n )\n self.client.force_login(self.superuser)\n\n self.config = Configuration.objects.create(\n key=\"test-key\",\n value=\"Initial value\",\n data_type=Configuration.DataType.TEXT,\n description=\"Initial description\",\n )\n self.url = reverse(\n \"admin:configuration_configuration_change\", args=[self.config.pk]\n )\n\n def test_change_view_initial_get(self):\n response = self.client.get(self.url)\n self.assertEqual(response.status_code, 200)\n self.assertContains(response, \"test-key\")\n self.assertContains(response, \"Initial value\")\n\n def test_save_triggers_confirmation(self):\n response = self.client.post(\n self.url,\n {\n \"key\": self.config.key,\n \"value\": \"Updated value\",\n \"data_type\": Configuration.DataType.TEXT,\n \"description\": \"Updated description\",\n },\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(response, \"admin/configuration_confirm_update.html\")\n self.assertContains(response, \"Confirm Update of Configuration\")\n\n def test_confirm_save_updates_object(self):\n # Step 1: post to trigger confirmation\n confirmation_response = self.client.post(\n self.url,\n {\n \"key\": self.config.key,\n \"value\": \"Updated value\",\n \"data_type\": Configuration.DataType.TEXT,\n \"description\": \"Updated description\",\n },\n )\n self.assertEqual(confirmation_response.status_code, 200)\n self.assertTemplateUsed(\n confirmation_response, \"admin/configuration_confirm_update.html\"\n )\n\n # Step 2: confirm the update\n confirm_response = self.client.post(\n self.url,\n {\n \"_confirm_update\": \"1\",\n \"key\": self.config.key,\n \"value\": \"Updated value\",\n \"data_type\": Configuration.DataType.TEXT,\n \"description\": \"Updated description\",\n },\n follow=True,\n )\n\n changelist_url = reverse(\"admin:configuration_configuration_changelist\")\n self.assertRedirects(confirm_response, changelist_url)\n self.assertContains(confirm_response, \"Configuration updated and cached.\")\n self.config.refresh_from_db()\n self.assertEqual(self.config.value, \"Updated value\")\n self.assertEqual(self.config.description, \"Updated description\")\n\n def test_cancel_preserves_input(self):\n # Step 1: post to trigger confirmation\n confirmation_response = self.client.post(\n self.url,\n {\n \"key\": self.config.key,\n \"value\": \"New value\",\n \"data_type\": Configuration.DataType.TEXT,\n \"description\": \"Changed description\",\n },\n )\n self.assertEqual(confirmation_response.status_code, 200)\n\n # Step 2: simulate \"Cancel\" by posting with cancel_update\n cancel_response = self.client.post(\n self.url,\n {\n \"cancel_update\": \"1\",\n \"key\": self.config.key,\n \"value\": \"New value\",\n \"data_type\": Configuration.DataType.TEXT,\n \"description\": \"Changed description\",\n },\n )\n\n self.assertEqual(cancel_response.status_code, 200)\n self.assertContains(cancel_response, \"Changed description\")\n self.assertContains(cancel_response, \"New value\")\n\n self.config.refresh_from_db()\n self.assertEqual(self.config.value, \"Initial value\")\n self.assertEqual(self.config.description, \"Initial description\")\n\n def test_confirm_save_with_invalid_form(self):\n # Step 1: trigger confirmation with valid initial post\n self.client.post(\n self.url,\n {\n \"key\": self.config.key,\n \"value\": \"value\",\n \"data_type\": Configuration.DataType.TEXT,\n \"description\": \"desc\",\n },\n )\n\n # Step 2: confirm with missing required field (invalid POST)\n response = self.client.post(\n self.url,\n {\n \"_confirm_update\": \"1\",\n # Omit 'key' which is required\n \"value\": \"value\",\n \"data_type\": Configuration.DataType.TEXT,\n \"description\": \"desc\",\n },\n follow=True,\n )\n\n self.assertEqual(response.status_code, 200)\n self.assertContains(response, \"Invalid data on confirmation.\")\n self.config.refresh_from_db()\n self.assertEqual(self.config.value, \"Initial value\") # unchanged\n\n def test_get_value_failure_on_confirmation(self):\n # Create a config with data_type=JSON and invalid JSON\n config = Configuration.objects.create(\n key=\"bad-json-key\",\n value=\"Not JSON\",\n data_type=Configuration.DataType.JSON,\n description=\"desc\",\n )\n url = reverse(\"admin:configuration_configuration_change\", args=[config.pk])\n\n # Initial POST with invalid JSON triggers get_value failure\n response = self.client.post(\n url,\n {\n \"key\": config.key,\n \"value\": \"Still not JSON\",\n \"data_type\": Configuration.DataType.JSON,\n \"description\": \"desc\",\n },\n follow=True,\n )\n\n self.assertEqual(response.status_code, 200)\n self.assertContains(response, \"Validation failed:\")\n\n def test_first_post_invalid_form(self):\n response = self.client.post(\n self.url,\n {\n \"key\": \"\", # key is required, so this makes the form invalid\n \"value\": \"Some value\",\n \"data_type\": Configuration.DataType.TEXT,\n \"description\": \"Bad post\",\n },\n )\n self.assertEqual(response.status_code, 200)\n self.assertTemplateUsed(response, \"admin/change_form.html\")\n self.assertContains(response, \"This field is required.\")\n"
},
{
"path": "configuration/tests/test_models.py",
"content": "import json\n\nfrom django.core.cache import caches\nfrom django.test import TestCase\n\nfrom configuration.models import Configuration\n\n\nclass TestConfiguration(TestCase):\n def setUp(self):\n caches[\"configuration_cache\"].clear()\n\n def test_str(self):\n config = Configuration.objects.create(\n key=\"test-key\", value=\"Test value\", data_type=Configuration.DataType.TEXT\n )\n self.assertEqual(str(config), \"test-key\")\n\n def test_text(self):\n config = Configuration.objects.create(\n key=\"test-key\", value=\"Test value\", data_type=Configuration.DataType.TEXT\n )\n self.assertEqual(config.get_value(), \"Test value\")\n\n config2 = Configuration.objects.create(\n key=\"test-key2\", value=\"\", data_type=Configuration.DataType.TEXT\n )\n self.assertEqual(config2.get_value(), \"\")\n\n config3 = Configuration.objects.create(\n key=\"test-key3\",\n value='{\"key\" : \"value\"}',\n data_type=Configuration.DataType.TEXT,\n )\n self.assertEqual(config3.get_value(), '{\"key\" : \"value\"}')\n\n def test_number(self):\n config = Configuration.objects.create(\n key=\"test-key\", value=\"100\", data_type=Configuration.DataType.NUMBER\n )\n self.assertEqual(config.get_value(), 100)\n\n config2 = Configuration.objects.create(\n key=\"test-key2\", value=\"100.12\", data_type=Configuration.DataType.NUMBER\n )\n self.assertEqual(config2.get_value(), 100.12)\n\n config3 = Configuration.objects.create(\n key=\"test-key3\", value=\"Test value\", data_type=Configuration.DataType.NUMBER\n )\n self.assertEqual(config3.get_value(), 0)\n\n config4 = Configuration.objects.create(\n key=\"test-key4\", value=\"\", data_type=Configuration.DataType.NUMBER\n )\n self.assertEqual(config4.get_value(), 0)\n\n def test_boolean(self):\n config = Configuration.objects.create(\n key=\"test-key\", value=\"True\", data_type=Configuration.DataType.BOOLEAN\n )\n self.assertEqual(config.get_value(), True)\n\n config2 = Configuration.objects.create(\n key=\"test-key2\", value=\"true\", data_type=Configuration.DataType.BOOLEAN\n )\n self.assertEqual(config2.get_value(), True)\n\n config3 = Configuration.objects.create(\n key=\"test-key3\", value=\"TrUe\", data_type=Configuration.DataType.BOOLEAN\n )\n self.assertEqual(config3.get_value(), True)\n\n config4 = Configuration.objects.create(\n key=\"test-key4\", value=\"\", data_type=Configuration.DataType.BOOLEAN\n )\n self.assertEqual(config4.get_value(), False)\n\n config5 = Configuration.objects.create(\n key=\"test-key5\", value=\"1\", data_type=Configuration.DataType.BOOLEAN\n )\n self.assertEqual(config5.get_value(), False)\n\n config6 = Configuration.objects.create(\n key=\"test-key6\",\n value=\"Test value\",\n data_type=Configuration.DataType.BOOLEAN,\n )\n self.assertEqual(config6.get_value(), False)\n\n def test_json(self):\n config = Configuration.objects.create(\n key=\"test-key\", value=\"true\", data_type=Configuration.DataType.JSON\n )\n self.assertEqual(config.get_value(), True)\n\n config2 = Configuration.objects.create(\n key=\"test-key2\", value=\"True\", data_type=Configuration.DataType.JSON\n )\n self.assertRaises(json.decoder.JSONDecodeError, config2.get_value)\n\n config3 = Configuration.objects.create(\n key=\"test-key3\",\n value='{\"key\" : \"value\"}',\n data_type=Configuration.DataType.JSON,\n )\n self.assertEqual(config3.get_value(), {\"key\": \"value\"})\n\n config4 = Configuration.objects.create(\n key=\"test-key4\", value=\"\", data_type=Configuration.DataType.JSON\n )\n self.assertRaises(json.decoder.JSONDecodeError, config4.get_value)\n\n config5 = Configuration.objects.create(\n key=\"test-key5\", value=\"1\", data_type=Configuration.DataType.JSON\n )\n self.assertEqual(config5.get_value(), 1)\n\n config6 = Configuration.objects.create(\n key=\"test-key6\", value=\"Test value\", data_type=Configuration.DataType.JSON\n )\n self.assertRaises(json.decoder.JSONDecodeError, config6.get_value)\n\n def test_html(self):\n config = Configuration.objects.create(\n key=\"test-key\", value=\"Test value\", data_type=Configuration.DataType.HTML\n )\n self.assertEqual(config.get_value(), \"Test value\")\n\n config2 = Configuration.objects.create(\n key=\"test-key2\", value=\"\", data_type=Configuration.DataType.HTML\n )\n self.assertEqual(config2.get_value(), \"\")\n\n config3 = Configuration.objects.create(\n key=\"test-key3\",\n value='{\"key\" : \"value\"}',\n data_type=Configuration.DataType.HTML,\n )\n self.assertEqual(config3.get_value(), '{\"key\" : \"value\"}')\n\n config4 = Configuration.objects.create(\n key=\"test-key4\",\n value=\"Test value
\",\n data_type=Configuration.DataType.HTML,\n )\n self.assertEqual(config4.get_value(), \"Test value
\")\n\n config5 = Configuration.objects.create(\n key=\"test-key5\",\n value=\"{% configuration_value 'test-key' %}
\",\n data_type=Configuration.DataType.HTML,\n )\n self.assertEqual(config5.get_value(), \"Test value
\")\n\n config6 = Configuration.objects.create(\n key=\"test-key6\",\n value=\"{% url 'homepage' %}\",\n data_type=Configuration.DataType.HTML,\n )\n self.assertEqual(config6.get_value(), \"/\")\n\n def test_rate(self):\n # Valid rates\n config1 = Configuration.objects.create(\n key=\"test-key1\", value=\"1/s\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config1.get_value(), \"1/s\")\n\n config2 = Configuration.objects.create(\n key=\"test-key2\", value=\"100/m\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config2.get_value(), \"100/m\")\n\n config3 = Configuration.objects.create(\n key=\"test-key3\", value=\"50/h\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config3.get_value(), \"50/h\")\n\n config4 = Configuration.objects.create(\n key=\"test-key4\", value=\"1000/d\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config4.get_value(), \"1000/d\")\n\n # Invalid formats\n config5 = Configuration.objects.create(\n key=\"test-key5\", value=\"5/hour\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config5.get_value(), \"\")\n\n config6 = Configuration.objects.create(\n key=\"test-key6\", value=\"ten/m\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config6.get_value(), \"\")\n\n config7 = Configuration.objects.create(\n key=\"test-key7\", value=\"10\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config7.get_value(), \"\")\n\n config8 = Configuration.objects.create(\n key=\"test-key8\", value=\"10/\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config8.get_value(), \"\")\n\n config9 = Configuration.objects.create(\n key=\"test-key9\", value=\"/m\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config9.get_value(), \"\")\n\n # Zero and negative values\n config10 = Configuration.objects.create(\n key=\"test-key10\", value=\"0/s\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config10.get_value(), \"\")\n\n config11 = Configuration.objects.create(\n key=\"test-key11\", value=\"-5/m\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config11.get_value(), \"\")\n\n # Empty value\n config12 = Configuration.objects.create(\n key=\"test-key12\", value=\"\", data_type=Configuration.DataType.RATE\n )\n self.assertEqual(config12.get_value(), \"\")\n"
},
{
"path": "configuration/tests/test_signals.py",
"content": "from django.core.cache import caches\nfrom django.test import TestCase\n\nfrom configuration.models import Configuration\n\n\nclass TestConfigurationSignal(TestCase):\n def setUp(self):\n caches[\"configuration_cache\"].clear()\n\n def test_signal_caches_valid_value(self):\n Configuration.objects.create(\n key=\"signal-key\",\n value=\"42\",\n data_type=Configuration.DataType.NUMBER,\n )\n self.assertEqual(caches[\"configuration_cache\"].get(\"config_signal-key\"), 42)\n\n def test_signal_does_not_cache_invalid_json(self):\n Configuration.objects.create(\n key=\"signal-json-invalid\",\n value=\"not valid json\",\n data_type=Configuration.DataType.JSON,\n )\n # Should not raise, but value should not be cached\n self.assertIsNone(\n caches[\"configuration_cache\"].get(\"config_signal-json-invalid\")\n )\n"
},
{
"path": "configuration/tests/test_templatetags.py",
"content": "from django.test import TestCase\n\nfrom configuration.models import Configuration\nfrom configuration.templatetags.configuration_tags import configuration_value\n\n\nclass TestTemplatetags(TestCase):\n def test_configuration_value(self):\n Configuration.objects.create(\n key=\"test-key\", value=\"Test value\", data_type=Configuration.DataType.TEXT\n )\n self.assertEqual(configuration_value(\"test-key\"), \"Test value\")\n\n Configuration.objects.create(\n key=\"test-key2\", value=\"100\", data_type=Configuration.DataType.NUMBER\n )\n self.assertEqual(configuration_value(\"test-key2\"), 100)\n\n Configuration.objects.create(\n key=\"test-key3\", value=\"TrUe\", data_type=Configuration.DataType.BOOLEAN\n )\n self.assertEqual(configuration_value(\"test-key3\"), True)\n\n Configuration.objects.create(\n key=\"test-key4\", value=\"Test value\", data_type=Configuration.DataType.JSON\n )\n # This raises an exception due to invalid JSON, but the template tag should\n # catch and return a blank string instead\n self.assertEqual(configuration_value(\"test-key4\"), \"\")\n\n Configuration.objects.create(\n key=\"test-key5\",\n value='{\"key\" : \"value\"}',\n data_type=Configuration.DataType.JSON,\n )\n self.assertEqual(configuration_value(\"test-key5\"), {\"key\": \"value\"})\n\n Configuration.objects.create(\n key=\"test-key6\",\n value=\"{% configuration_value 'test-key' %}
\",\n data_type=Configuration.DataType.HTML,\n )\n self.assertEqual(configuration_value(\"test-key6\"), \"Test value
\")\n"
},
{
"path": "configuration/tests/test_utils.py",
"content": "import json\n\nfrom django.core.cache import caches\nfrom django.test import TestCase\n\nfrom configuration.models import Configuration\nfrom configuration.utils import (\n CONFIGURATION_KEY_PREFIX,\n cache_configuration_value,\n configuration_value,\n)\n\n\nclass TestConfigurationUtils(TestCase):\n def setUp(self):\n self.cache = caches[\"configuration_cache\"]\n self.cache.clear()\n\n def test_configuration_value(self):\n Configuration.objects.create(\n key=\"test-key\", value=\"Test value\", data_type=Configuration.DataType.TEXT\n )\n self.assertEqual(configuration_value(\"test-key\"), \"Test value\")\n\n Configuration.objects.create(\n key=\"test-key2\", value=\"100\", data_type=Configuration.DataType.NUMBER\n )\n self.assertEqual(configuration_value(\"test-key2\"), 100)\n\n Configuration.objects.create(\n key=\"test-key3\", value=\"TrUe\", data_type=Configuration.DataType.BOOLEAN\n )\n self.assertEqual(configuration_value(\"test-key3\"), True)\n\n Configuration.objects.create(\n key=\"test-key4\", value=\"\", data_type=Configuration.DataType.JSON\n )\n self.assertRaises(\n json.decoder.JSONDecodeError, configuration_value, \"test-key4\"\n )\n\n Configuration.objects.create(\n key=\"test-key5\",\n value='{\"key\" : \"value\"}',\n data_type=Configuration.DataType.JSON,\n )\n self.assertEqual(configuration_value(\"test-key5\"), {\"key\": \"value\"})\n\n Configuration.objects.create(\n key=\"test-key6\",\n value=\"{% configuration_value 'test-key' %}
\",\n data_type=Configuration.DataType.HTML,\n )\n self.assertEqual(configuration_value(\"test-key6\"), \"Test value
\")\n\n\nclass TestCacheConfigurationValue(TestCase):\n def setUp(self):\n self.cache = caches[\"configuration_cache\"]\n self.cache.clear()\n\n def test_explicit_value_is_cached(self):\n key = \"explicit-key\"\n cached = cache_configuration_value(key, 123)\n self.assertEqual(cached, 123)\n self.assertEqual(\n self.cache.get(f\"{CONFIGURATION_KEY_PREFIX}_{key}\"),\n 123,\n )\n\n def test_value_is_fetched_from_model_if_not_supplied(self):\n Configuration.objects.create(\n key=\"fetched-key\", value=\"true\", data_type=Configuration.DataType.BOOLEAN\n )\n cached = cache_configuration_value(\"fetched-key\")\n self.assertEqual(cached, True)\n self.assertEqual(\n self.cache.get(f\"{CONFIGURATION_KEY_PREFIX}_fetched-key\"),\n True,\n )\n"
},
{
"path": "configuration/tests/test_validation.py",
"content": "from django.core.exceptions import ValidationError\nfrom django.test import TestCase\n\nfrom configuration.validation import validate_rate\n\n\nclass TestValidation(TestCase):\n def test_valid_rates(self):\n self.assertEqual(validate_rate(\"1/s\"), \"1/s\")\n self.assertEqual(validate_rate(\"10/m\"), \"10/m\")\n self.assertEqual(validate_rate(\"100/h\"), \"100/h\")\n self.assertEqual(validate_rate(\"1000/d\"), \"1000/d\")\n\n def test_rate_stripping_whitespace(self):\n # Leading/trailing spaces\n self.assertEqual(validate_rate(\" 10/m \"), \"10/m\")\n self.assertEqual(validate_rate(\"\\t10/m\"), \"10/m\")\n self.assertEqual(validate_rate(\"10/m\\t\"), \"10/m\")\n self.assertEqual(validate_rate(\"\\n10/m\\n\"), \"10/m\")\n self.assertEqual(validate_rate(\" \\n\\t10/m\\t\\n \"), \"10/m\")\n\n # Internal whitespace is not allowed and should still raise\n with self.assertRaises(ValidationError):\n validate_rate(\"10 /m\")\n\n with self.assertRaises(ValidationError):\n validate_rate(\"10/ m\")\n\n with self.assertRaises(ValidationError):\n validate_rate(\"10 / m\")\n\n def test_non_string_input(self):\n with self.assertRaises(ValidationError):\n validate_rate(10)\n\n with self.assertRaises(ValidationError):\n validate_rate(None)\n\n with self.assertRaises(ValidationError):\n validate_rate([\"5/m\"])\n\n def test_invalid_format(self):\n with self.assertRaises(ValidationError):\n validate_rate(\"10\") # no unit\n\n with self.assertRaises(ValidationError):\n validate_rate(\"10/min\") # full word\n\n with self.assertRaises(ValidationError):\n validate_rate(\"ten/m\") # non-numeric\n\n with self.assertRaises(ValidationError):\n validate_rate(\"10/\") # missing unit\n\n with self.assertRaises(ValidationError):\n validate_rate(\"/m\") # missing number\n\n # This is now valid due to stripping\n self.assertEqual(validate_rate(\"10/m\\n\"), \"10/m\")\n\n with self.assertRaises(ValidationError):\n validate_rate(\"10/m/extra\") # too many parts\n\n def test_zero_or_negative_values(self):\n with self.assertRaises(ValidationError):\n validate_rate(\"0/s\")\n\n with self.assertRaises(ValidationError):\n validate_rate(\"-5/m\")\n\n def test_invalid_unit(self):\n with self.assertRaises(ValidationError):\n validate_rate(\"10/w\") # unsupported unit\n\n with self.assertRaises(ValidationError):\n validate_rate(\"10/ms\") # unsupported unit\n\n with self.assertRaises(ValidationError):\n validate_rate(\"10/seconds\") # full unit\n"
},
{
"path": "configuration/utils.py",
"content": "from typing import Any\n\nfrom django.conf import settings\nfrom django.core.cache import caches\n\nfrom configuration.models import Configuration\n\nCONFIGURATION_KEY_PREFIX = \"config\"\n\n\ndef configuration_value(key: str) -> Any:\n \"\"\"\n Retrieve a configuration value by key with caching and type casting.\n\n Behavior:\n - Look up the value in the ``configuration_cache`` using a namespaced\n cache key.\n - If the value is missing, delegate to\n ``cache_configuration_value(key)`` to fetch, cast, cache, and return\n the value.\n - Casting is performed by ``Configuration.get_value()`` based on the\n instance's ``data_type``.\n\n Caching:\n - Values are stored in the cache alias ``configuration_cache``.\n - Cache entries expire according to\n ``settings.CONFIGURATION_CACHE_TIMEOUT``.\n\n Args:\n key (str): The configuration key to resolve.\n\n Returns:\n Any: The resolved and type-cast configuration value.\n\n Raises:\n Configuration.DoesNotExist: If the key is not present in the database\n when attempting to populate the cache.\n \"\"\"\n config_cache = caches[\"configuration_cache\"]\n cache_key = f\"{CONFIGURATION_KEY_PREFIX}_{key}\"\n value = config_cache.get(cache_key)\n\n if value is None:\n value = cache_configuration_value(key)\n\n return value\n\n\ndef cache_configuration_value(key: str, value: Any | None = None) -> Any:\n \"\"\"\n Populate or refresh the cached value for a configuration key.\n\n Behavior:\n - If ``value`` is ``None``, fetch the ``Configuration`` by ``key``,\n cast it via ``get_value()``, and cache the result.\n - If ``value`` is provided, cache that value directly.\n - Always write to the ``configuration_cache`` using the configured\n ``settings.CONFIGURATION_CACHE_TIMEOUT``.\n\n Args:\n key (str): The configuration key to cache.\n value (Any | None): An explicit value to cache. If ``None``, the value\n is loaded from the database and cast via ``get_value()``.\n\n Returns:\n Any: The value that was stored in the cache.\n\n Raises:\n Configuration.DoesNotExist: If ``value`` is ``None`` and there is no\n ``Configuration`` row with the given key.\n \"\"\"\n config_cache = caches[\"configuration_cache\"]\n cache_key = f\"{CONFIGURATION_KEY_PREFIX}_{key}\"\n\n if value is None:\n config = Configuration.objects.get(key=key)\n value = config.get_value()\n\n config_cache.set(cache_key, value, timeout=settings.CONFIGURATION_CACHE_TIMEOUT)\n return value\n"
},
{
"path": "configuration/validation.py",
"content": "import re\n\nfrom django.core.exceptions import ValidationError\n\nRATE_LIMIT_PATTERN = re.compile(r\"^\\d+/(s|m|h|d)$\")\n\n\ndef validate_rate(rate: str) -> str:\n \"\"\"\n Validate that a rate string matches the expected pattern like '10/m'.\n\n Behavior:\n - Strip leading and trailing whitespace.\n - Require the format '/' where unit is one of\n 's', 'm', 'h', or 'd' (seconds, minutes, hours, days).\n - Return the cleaned string unchanged if valid.\n\n Args:\n rate (str): The candidate rate string to validate.\n\n Returns:\n str: The cleaned rate string if valid.\n\n Raises:\n ValidationError: If the input is not a string, if the format does not\n match the required pattern, or if the integer portion is less than\n or equal to zero.\n \"\"\"\n if not isinstance(rate, str):\n raise ValidationError(\"Rate limit must be a string.\")\n\n rate = rate.strip()\n\n if not RATE_LIMIT_PATTERN.match(rate):\n raise ValidationError(\"Invalid rate limit format. Use '/'.\")\n\n count, unit = rate.split(\"/\")\n if int(count) <= 0:\n raise ValidationError(\"Rate limit count must be greater than 0.\")\n\n return rate\n"
},
{
"path": "configuration/views.py",
"content": "# Create your views here.\n"
},
{
"path": "db_scripts/Dockerfile",
"content": "# Base layer with all tools\nFROM public.ecr.aws/amazonlinux/amazonlinux:2023-minimal AS base\n\n# Trusted CA for proxy\nRUN curl -fsO --output-dir /etc/pki/ca-trust/source/anchors/ http://crl.loc.gov/LOC-ROOT-CA-1.crt \\\n && update-ca-trust\n\n# Install tools once\nRUN dnf -y upgrade && \\\n dnf -y install postgresql15.x86_64 awscli-2.noarch && \\\n dnf -y clean all\n\n# Logic for Dump\nFROM base AS dump\nCOPY dump.sh .\nRUN chmod +x dump.sh\nCMD [\"./dump.sh\"]\n\n# Logic for Restore\nFROM base AS restore\nCOPY restore.sh .\nRUN chmod +x restore.sh\nCMD [\"./restore.sh\"]\n"
},
{
"path": "db_scripts/dump.sh",
"content": "#!/bin/bash\n\nset -eu -o pipefail\n\nif [[ -z \"${ENV_NAME}\" ]]; then\n echo \"ENV_NAME must be set prior to running this script.\"\n exit 1\nfi\n\nif [ $ENV_NAME != \"prod\" ]; then\n echo \"This script should only be run in the production environment.\"\n exit 1\nfi\n\nTODAY=$(date +%Y%m%d)\nif [[ \"$TODAY\" =~ (0101|0401|0701|1001)$ ]]; then\n TAGVALUE=\"true\"\nelse\n TAGVALUE=\"false\"\nfi\nPOSTGRESQL_PW=\"$(aws secretsmanager get-secret-value --region us-east-1 --secret-id crowd/${ENV_NAME}/DB/MasterUserPassword | python3 -c 'import json,sys;Secret=json.load(sys.stdin);SecretString=json.loads(Secret[\"SecretString\"]);print(SecretString[\"password\"])')\"\nPOSTGRESQL_HOST=\"$(aws ssm get-parameter --region us-east-1 --name /concordia/${ENV_NAME}/db.url | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput[\"Parameter\"];print(Parameter[\"Value\"])')\"\nDUMP_FILE=concordia.dmp\n\necho \"${POSTGRESQL_HOST}:5432:*:concordia:${POSTGRESQL_PW}\" > ~/.pgpass\nchmod 600 ~/.pgpass\n\npg_dump -Fc --no-acl -U concordia -h \"${POSTGRESQL_HOST}\" concordia -f \"${DUMP_FILE}\"\n\nif [ -s $DUMP_FILE ]; then\n aws s3 cp \"${DUMP_FILE}\" \"s3://crowd-deployment/database-dumps/concordia.${TODAY}.dmp\"\n aws s3 cp \"${DUMP_FILE}\" s3://crowd-deployment/database-dumps/concordia.latest.dmp\n aws s3api put-object-tagging --bucket 'crowd-deployment' --key database-dumps/concordia.${TODAY}.dmp --tagging '{\"TagSet\": [{ \"Key\": \"first-dmp-of-quarter\", \"Value\": \"'${TAGVALUE}'\" }]}'\n aws s3api put-object-tagging --bucket 'crowd-deployment' --key database-dumps/concordia.latest.dmp --tagging '{\"TagSet\": [{ \"Key\": \"first-dmp-of-quarter\", \"Value\": \"'${TAGVALUE}'\" }]}'\nfi\necho $?\n"
},
{
"path": "db_scripts/restore.sh",
"content": "#!/bin/bash\n\nset -eu -o pipefail\n\nexport PATH=$HOME/.local/bin:$PATH\n\nif [[ -z \"${ENV_NAME}\" ]]; then\n echo \"ENV_NAME must be set prior to running this script.\"\n exit 1\nfi\n\nif [ $ENV_NAME = \"prod\" ]; then\n echo \"This script should not be run in the production environment.\"\n exit 1\nfi\n\nPOSTGRESQL_PW=\"$(aws secretsmanager get-secret-value --region us-east-1 --secret-id crowd/${ENV_NAME}/DB/MasterUserPassword | python3 -c 'import json,sys;Secret=json.load(sys.stdin);SecretString=json.loads(Secret[\"SecretString\"]);print(SecretString[\"password\"])')\"\nPOSTGRESQL_HOST=\"$(aws ssm get-parameter --region us-east-1 --name /concordia/${ENV_NAME}/db.url | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput[\"Parameter\"];print(Parameter[\"Value\"])')\"\nDUMP_FILE=concordia.dmp\n\naws s3 cp s3://crowd-deployment/database-dumps/concordia.latest.dmp ${DUMP_FILE}\n\necho \"${POSTGRESQL_HOST}:5432:*:concordia:${POSTGRESQL_PW}\" > ~/.pgpass\nchmod 600 ~/.pgpass\n\naws s3 sync s3://crowd-content s3://crowd-${ENV_NAME}-content --delete\n\npsql -U concordia -h \"$POSTGRESQL_HOST\" -d postgres -c \"select pg_terminate_backend(pid) from pg_stat_activity where datname='concordia';\"\npsql -U concordia -h \"$POSTGRESQL_HOST\" -d postgres -c \"drop database concordia with (force);\"\npg_restore --create -U concordia -h \"${POSTGRESQL_HOST}\" -Fc --dbname=postgres --no-owner --no-acl \"${DUMP_FILE}\"\nRETURNCODE=$?\necho $RETURNCODE\n\nif [ $RETURNCODE = 0 ] && [ $ENV_NAME = \"test\" ]; then\n ECS_SERVICE=\"$(aws ecs list-services --region us-east-1 --cluster crowd-${ENV_NAME} | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput[\"serviceArns\"];print(Parameter[0].split(\"/\")[2])')\"\n\n # If a feature branch env is running the number of services in the test cluster increases.\n NUMBER_OF_SERVICES=\"$(aws ecs list-services --region us-east-1 --cluster crowd-test | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput[\"serviceArns\"];print(len(Parameter))')\"\n if [ $NUMBER_OF_SERVICES = 3 ];then\n # Normal\n ECS_SERVICE_2=\"$(aws ecs list-services --region us-east-1 --cluster crowd-${ENV_NAME} | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput[\"serviceArns\"];print(Parameter[2].split(\"/\")[2])')\"\n else\n # Feature branch env exists.\n ECS_SERVICE_2=\"$(aws ecs list-services --region us-east-1 --cluster crowd-${ENV_NAME} | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput[\"serviceArns\"];print(Parameter[3].split(\"/\")[2])')\"\n fi\n\n aws ecs update-service --region us-east-1 --force-new-deployment --cluster crowd-${ENV_NAME} --service ${ECS_SERVICE}\n aws ecs update-service --region us-east-1 --force-new-deployment --cluster crowd-${ENV_NAME} --service ${ECS_SERVICE_2}\nelif [ $RETURNCODE = 0 ]; then\n ECS_SERVICE=\"$(aws ecs list-services --region us-east-1 --cluster crowd-${ENV_NAME} | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput[\"serviceArns\"];print(Parameter[0].split(\"/\")[2])')\"\n aws ecs update-service --region us-east-1 --force-new-deployment --cluster crowd-${ENV_NAME} --service ${ECS_SERVICE}\nfi\n"
},
{
"path": "development/Containerfile",
"content": "FROM python:3.12-slim-bookworm\n\n# Major Node.js version to install (e.g., 20, 22). This is used to select the\n# NodeSource APT repository \"node_.x\".\nARG NODE_MAJOR=20\n\n# Define build-time arguments for UID and GID\nARG USERNAME\nARG UID\nARG GID\n\nENV DEBIAN_FRONTEND=\"noninteractive\"\n\n# Create the group and user with specified UID/GID\nRUN groupadd -g $GID $USERNAME && \\\n useradd -m -u $UID -g $GID -s /bin/bash $USERNAME\n\n# Bootstrap minimal tooling needed later in the build:\n# - curl: download files/keys\n# - ca-certificates: validate HTTPS endpoints\n# - gnupg: import and dearmor APT repository signing keys\nRUN apt-get update -qy && apt-get install -qy curl ca-certificates gnupg\n\n# Ensure that the Library's certificate authority is trusted so the tampering\n# proxy will not break TLS validation. See\n# https://staff.loc.gov/wikis/display/SE/Configuring+HTTPS+clients+for+the+HTTPS+tampering+proxy.\n\nRUN curl -fso /etc/ssl/certs/LOC-ROOT-CA-1.crt http://crl.loc.gov/LOC-ROOT-CA-1.crt && openssl x509 -inform der -in /etc/ssl/certs/LOC-ROOT-CA-1.crt -outform pem -out /etc/ssl/certs/LOC-ROOT-CA-1.pem && c_rehash\n\n# Install Node.js via the NodeSource APT repository (manual setup; no setup\n# script). Debian bookworm ships Node 18; adding this repo allows installing a\n# newer major version (e.g., Node 20) via apt.\n#\n# This step:\n# - creates a dedicated keyring directory under /etc/apt/keyrings\n# - downloads and installs the NodeSource signing key into a keyring file\n# - registers the NodeSource repository for the selected Node.js major line\n#\n# Note: When installing Node.js from NodeSource, the `nodejs` package includes\n# npm (and npm comes with node-gyp), so there is no separate `npm` or\n# `node-gyp` APT package to install here.\n#\n# References: NodeSource \"Repository Manual Installation\" guide. https://github.com/nodesource/distributions/wiki/Repository-Manual-Installation\nRUN \\\n # Create a dedicated directory for third-party APT keyrings.\n mkdir -p /etc/apt/keyrings && \\\n # Download the NodeSource repository signing key and store it as a keyring\n # file that apt can use to verify NodeSource packages.\n curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \\\n | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \\\n # Register the NodeSource repository for the selected Node.js major version.\n # The \"signed-by=\" option scopes trust to just this repository entry.\n echo \"deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main\" \\\n > /etc/apt/sources.list.d/nodesource.list\n\nRUN apt-get update -qy && apt-get dist-upgrade -qy && apt-get install -o Dpkg::Options::='--force-confnew' -qy \\\n build-essential \\\n git \\\n libmemcached-dev \\\n # Pillow/Imaging: https://pillow.readthedocs.io/en/latest/installation.html#external-libraries\n libz-dev libfreetype6-dev \\\n libtiff-dev libjpeg-dev libopenjp2-7-dev libwebp-dev zlib1g-dev \\\n # Postgres client library to build psycopg\n libpq-dev \\\n locales \\\n # Weasyprint requirements\n libpango-1.0-0 libharfbuzz0b libpangoft2-1.0-0 \\\n # Tesseract\n tesseract-ocr tesseract-ocr-all \\\n # Selenium/Chrome/chromedriver requirements\n libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libxdamage1 libxkbcommon0 libasound2 libatspi2.0-0 \\\n # Additional tools for development\n nano bash-completion \\\n nodejs && apt-get -qy autoremove && apt-get -qy autoclean\n\nRUN locale-gen en_US.UTF-8\nENV LC_ALL=en_US.UTF-8\nENV LANG=en_US.UTF-8\nENV LANGUAGE=en_US.UTF-8\n\nENV PYTHONUNBUFFERED 1\nENV PYTHONPATH /workspace\n\nENV DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-concordia.settings_docker}\n\nRUN pip install --upgrade pip\nRUN pip install --no-cache-dir pre-commit\nRUN pip install --no-cache-dir pipenv\n\nRUN npm install --silent --global npm@10\n\n# Set the working directory and permissions\nWORKDIR /workspace\nCOPY . /workspace\nRUN chown -R $USERNAME:$USERNAME /workspace\n\n# Switch to the new user\nUSER $USERNAME\n\n# Set user's path to include local bin, which is where Python libraries will be installed\nENV PATH \"/home/${USERNAME}/.local/bin:$PATH\"\n\nRUN mkdir -p /workspace/logs\nRUN touch /workspace/logs/concordia.log\n\n# Front-end build and asset pipeline:\n# - update npm to a known major version\n# - install JS dependencies (production-only)\nRUN npm install --silent\n# Additional JS build step for Vite - compile scss, bundle, hash, and compress js files.\nRUN npm run build\n\nRUN pipenv install --system --dev --deploy && rm -rf ~/.cache/\n\nRUN pre-commit install-hooks\n\nEXPOSE 80\n\nCMD [\"bash\"]\n"
},
{
"path": "development/README.md",
"content": "# Concordia Development Containers\n\nThe files in this directory, `compose.yml` and `Containerfile`, have been created to better facilitate developing Concordia in a containerized environment. The files are compatible with both Docker and Podman, using docker-compose or podman-compose.\n\nThough newer versions of docker-compose and podman-compose support combining compose files and compose file overriding, the versions of these tools available on some current distributions (such as Red Hat Enterprise Linux 9) do not, so a singular compose file (`compose.yml`) with all the necessary settings is provided here.\n\n## Purpose\n\nThe intention of these files is to provide a usable development environment purely in containers.\n\nThe default configuration (`../docker-compose.yml`) creates a container environment that's not suitable for development. Primarily, it creates an container for Elasticsearch, which is often used except in production, and the `app` container is not configured well for development. It runs as root, which causes file permission issues, and runs the daphne asgi server, which can't be restarted without restarting the entire container (manually killing and starting daphne does not work, either, because that causes the container to shutdown).\n\nIn addition, the default container file for the `app` container (`../Dockerfile`) runs `../entrypoint.sh`, which does several things that are undesirable in a development environment that involves restarting containers regularly. It automatically generates and applies migrations, runs collectstatic and launches daphne. The development Containerfile instead simply launches a bash shell.\n\n## Configuration\n\nIn order to use `compose.yml` with your compose CLI tool of choice, you'll need to pass in the path, either through an environment variable or a command-line switch.\n\n```bash\npodman-compose -f development/compose.yml\n```\n\n```dotenv\nCOMPOSE_FILE=development/compose.yml\n```\n\nYour CLI tool should be executed in the concordia directory (`..`).\n\n### .env caveat\n\nNote that some versions of the tools (notably, podman-compose<=1.0.6) do not use .env files. If you wish to use environment variables, you'll need to inject those variables manually, such as by using a script that sources your .env file before executing podman-compose.\n\n## Development\n\nConfiguring for your environment when using containers follows the process in the [For Developers](../docs/for-developers.md) page, except that most of the work is done for you by `compose.yml` and `Containerfile`. Notably, you do not need to install any dependencies (except git and your compose CLI) on your host. You will still need to configure your .env file, but otherwise simply running `podman-compose up -d` or `docker-compose up -d`, with the proper COMPOSE_FILE configuration (and other additions to .env, see below), will create an app container with everything you need for development.\n\n### Additions to .env\n\nThere are a few additions to your .env file required to properly use the provided `compose.yml` and `Containerfile`:\n\n```dotenv\nCOMPOSE_FILE=development/compose.yml\nHOME_DIR=/home//\nAWS_SHARED_CREDENTIALS_FILE=/home//.aws/credentials\nCONTAINER_UID=\nCONTAINER_GID=\nCONTAINER_USERNAME=\n```\n\nThese values should be for the user account that you'll be doing development with, the same one that owns your local repository. This information is used to mount various necessary directories in the container, as well as configure the user account inside the container (to avoid running as root).\n\nThe last three settings can automatically be added to your .env file with the following scripts (executed in the directory with the .env):\n\n```bash\n#!/bin/bash\n\nENV_FILE=\".env\"\nBACKUP_FILE=\".env.bak\"\n\n# Create the .env file if it doesn't exist\ntouch \"$ENV_FILE\"\n\n# Backup the original .env once\ncp \"$ENV_FILE\" \"$BACKUP_FILE\"\n\n# Set the values\nNEW_UID=$(id -u)\nNEW_GID=$(id -g)\nNEW_USERNAME=$(whoami)\n\n# Function to add or update a key in the .env file\nupdate_env_var() {\n local key=\"$1\"\n local value=\"$2\"\n if grep -qE \"^${key}=\" \"$ENV_FILE\"; then\n sed -i \"s/^${key}=.*/${key}=${value}/\" \"$ENV_FILE\"\n else\n echo \"${key}=${value}\" >> \"$ENV_FILE\"\n fi\n}\n\n# Update the values\nupdate_env_var \"CONTAINER_UID\" \"$NEW_UID\"\nupdate_env_var \"CONTAINER_GID\" \"$NEW_GID\"\nupdate_env_var \"CONTAINER_USERNAME\" \"$NEW_USERNAME\"\n\necho \"Backup saved as $BACKUP_FILE\"\necho \".env updated with CONTAINER_UID=$NEW_UID, CONTAINER_GID=$NEW_GID, CONTAINER_USERNAME=$NEW_USERNAME\"\n```\n\n### Attaching to the app container\n\nOnce you've launched your containers, you can attach to a shell in the app container to perform development. Your compose CLI tool should provide a method for doing this.\n\n```bash\nsudo docker-compose exec -it app bash\n```\n\nNote that older versions of podman-compose do not properly pass switches to the underlying command, meaning the above won't work with those versions of podmon-compose. You can instead run it without the switches:\n\n```bash\nsudo podman-compose exec app bash\n```\n\nHowever, this has the disadvantage of not creating an interactive shell shell, which can cause issues with bash functionality. If you have a version of podman-compose with this limitation, the workaround is to use podman directly instead:\n\n```bash\nsudo podman exec -it concordia_app bash\n```\n\nYou have to use the full container name because compose.yml is not referenced when using podman directly.\n\n### Configuring for development\n\nYou will need to manually collect the static files before running the development server. This only needs to be done once after building the app container (and when changing static files in the future).\n\n```bash\nnpx vite build\npython manage.py collectstatic --no-post-process\n```\n\n### Launch the development server\n\nLaunching the development server is identical to launching it outside a container. The app container is configured to map port 8000 in the container to port 80 on the host:\n\n```bash\npython manage.py runserver 0.0.0.0:8000\n```\n\n### Committing changes\n\nGit and the Concordia precommit hooks are included in the app container. You can simply use git commands as normal inside /workspace in the container.\n"
},
{
"path": "development/compose.yml",
"content": "version: '3.6'\nservices:\n redis:\n container_name: concordia_redis\n restart: unless-stopped\n image: redis:latest\n hostname: redis\n ports:\n - 63791:6379\n volumes:\n - redis_volume:/data\n\n db:\n container_name: concordia_db\n restart: unless-stopped\n image: postgres:15\n environment:\n POSTGRES_PASSWORD: ${POSTGRESQL_PW}\n POSTGRES_USER: concordia\n POSTGRES_MULTIPLE_DATABASES: test_concordia\n ports:\n - 54323:5432\n volumes:\n - ../postgresql:/docker-entrypoint-initdb.d:z\n - db_volume:/var/lib/postgresql/data/\n\n app:\n container_name: concordia_app\n hostname: app\n restart: unless-stopped\n build:\n context: ..\n dockerfile: development/Containerfile\n args:\n UID: ${CONTAINER_UID}\n GID: ${CONTAINER_GID}\n USERNAME: ${CONTAINER_USERNAME}\n environment: &django-environment\n WAIT_HOSTS: db:5432, redis:6379\n POSTGRESQL_HOST: db\n POSTGRESQL_PW: ${POSTGRESQL_PW}\n CONCORDIA_ENVIRONMENT: development\n DJANGO_SETTINGS_MODULE: ${DJANGO_SETTINGS_MODULE:-concordia.settings_template}\n DEBUG: ${DEBUG:-}\n REDIS_ADDRESS: redis\n REDIS_PORT: 6379\n AWS_PROFILE: ${AWS_PROFILE}\n AWS_SHARED_CREDENTIALS_FILE: ${AWS_SHARED_CREDENTIALS_FILE}\n TURNSTILE_SITEKEY: ${TURNSTILE_SITEKEY:-1x00000000000000000000AA}\n TURNSTILE_SECRET: ${TURNSTILE_SECRET:-1x0000000000000000000000000000000AA}\n depends_on:\n - redis\n - db\n volumes:\n - ..:/workspace:z\n - ${HOME_DIR}/.aws:${HOME_DIR}/.aws:z\n - ${HOME_DIR}/.gitconfig:${HOME_DIR}/.gitconfig:z\n - ${HOME_DIR}/.ssh:${HOME_DIR}/.ssh:z\n - images_volume:/concordia_images\n networks:\n - default\n ports:\n - 80:8000\n stdin_open: true\n tty: true\n\n importer:\n container_name: concordia_importer\n hostname: importer\n restart: unless-stopped\n build:\n context: ..\n dockerfile: importer/Dockerfile\n environment: *django-environment\n depends_on:\n - redis\n - db\n networks:\n - default\n volumes:\n - ..:/app:z\n - ${HOME_DIR}/.aws:/root/.aws:z\n - images_volume:/concordia_images\n\n celerybeat:\n container_name: concordia_celerybeat\n hostname: celerybeat\n restart: unless-stopped\n build:\n context: ..\n dockerfile: celerybeat/Dockerfile\n environment: *django-environment\n depends_on:\n - redis\n - db\n networks:\n - default\n\nvolumes:\n db_volume:\n images_volume:\n redis_volume:\n\nnetworks:\n default:\n"
},
{
"path": "docker-compose.yml",
"content": "version: '3.6'\nservices:\n opensearch-node:\n image: public.ecr.aws/opensearchproject/opensearch:1\n container_name: opensearch-node\n environment:\n - cluster.name=opensearch-cluster\n - node.name=opensearch-node\n - discovery.type=single-node\n - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping\n - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m # minimum and maximum Java heap size, recommend setting both to 50% of system RAM\n - 'DISABLE_INSTALL_DEMO_CONFIG=true' # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch\n - 'DISABLE_SECURITY_PLUGIN=true' # initial local setup - Disables security plugin\n ulimits:\n memlock:\n soft: -1\n hard: -1\n nofile:\n soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems\n hard: 65536\n volumes:\n - opensearch-data:/usr/share/opensearch/data\n ports:\n - 9200:9200 # REST API\n - 9600:9600 # Performance Analyzer\n networks:\n - default\n\n opensearch-dashboards:\n image: public.ecr.aws/opensearchproject/opensearch-dashboards:1\n container_name: opensearch-dashboards\n ports:\n - 5601:5601\n expose:\n - '5601'\n environment:\n - 'OPENSEARCH_HOSTS=http://opensearch-node:9200'\n - 'DISABLE_SECURITY_DASHBOARDS_PLUGIN=true' # disables security dashboards plugin in OpenSearch Dashboards\n networks:\n - default\n\n redis:\n restart: unless-stopped\n image: redis:latest\n hostname: redis\n ports:\n - 6379:6379\n volumes:\n - redis_volume:/data\n\n db:\n restart: unless-stopped\n image: postgres:15\n environment:\n POSTGRES_PASSWORD: ${POSTGRESQL_PW}\n POSTGRES_USER: concordia\n POSTGRES_MULTIPLE_DATABASES: test_concordia\n ports:\n - 5432:5432\n volumes:\n - ./postgresql:/docker-entrypoint-initdb.d\n - db_volume:/var/lib/postgresql/data/\n\n app:\n restart: unless-stopped\n build: .\n env_file:\n - .env\n environment: &django-environment\n WAIT_HOSTS: db:5432, redis:6379\n POSTGRESQL_HOST: db\n POSTGRESQL_PW: ${POSTGRESQL_PW}\n CONCORDIA_ENVIRONMENT: development\n DJANGO_SETTINGS_MODULE: ${DJANGO_SETTINGS_MODULE:-concordia.settings_docker}\n DEBUG: ${DEBUG:-}\n REDIS_ADDRESS: redis\n REDIS_PORT: 6379\n depends_on:\n - redis\n - db\n volumes:\n - .:/app\n - images_volume:/concordia_images\n networks:\n - default\n ports: # if running locally use 80:80, if running in local container use 8000:80\n - 80:80\n\n importer:\n restart: unless-stopped\n build:\n context: .\n dockerfile: importer/Dockerfile\n environment: *django-environment\n depends_on:\n - redis\n - db\n networks:\n - default\n volumes:\n - images_volume:/concordia_images\n\n celerybeat:\n restart: unless-stopped\n build:\n context: .\n dockerfile: celerybeat/Dockerfile\n environment: *django-environment\n depends_on:\n - redis\n - db\n networks:\n - default\n\nvolumes:\n db_volume:\n images_volume:\n redis_volume:\n opensearch-data:\n\nnetworks:\n default:\n"
},
{
"path": "docs/accessibility-goals.md",
"content": "# Accessibility Goals\n\n## Background Information\n\n- Microsoft has a site with a number of resources detailing their βInclusive Designβ strategy: https://www.microsoft.com/design/inclusive/\n\n Of note is the [βInclusive 101β PDF](https://download.microsoft.com/download/b/0/d/b0d4bf87-09ce-4417-8f28-d60703d672ed/inclusive_toolkit_manual_final.pdf) covering the range of factors which designs should take into account and common strategies.\n\n [Kill Your Personas: How persona spectrums champion real user needs](https://medium.com/microsoft-design/kill-your-personas-1c332d4908cc)\n has a longer discussion about avoiding binary thinking when designing\n\n## Desired Outcomes by Activity\n\nWe have desired goals for each category of activity on the site\n\n### Transcription\n\n- Users with mobility restrictions should be able to complete the process\n- Users with color vision impairments should be able to complete the process except for specific pages where the content is inherently inaccessible until https://github.com/LibraryOfCongress/concordia/issues/666 is complete\n- Users with significant vision impairments should be able to learn the nature of the task and why it is difficult to continue\n\n### Review\n\n- Users with mobility restrictions should be able to complete the process\n- Users with color vision impairments should be able to complete the process except for specific pages where the content is inherently inaccessible until https://github.com/LibraryOfCongress/concordia/issues/666 is complete\n- Users with significant vision impairments should be able to learn the nature of the task and why it is difficult to continue\n\n### Final results\n\n- The transcribed text should be accessible to everyone\n\n### Project Information\n\n- Information about the project should be accessible to everyone\n- Users should be able to easily learn the level of difficulty for their participation in particular activities\n"
},
{
"path": "docs/accessibility-techniques.md",
"content": "# Accessibility Techniques\n\n## Background Information\n\n- 18F has a detailed U.S. government-focused guide: https://accessibility.18f.gov/\n- The BBC has their comprehensive guide: https://bbc.github.io/accessibility-news-and-you/\n\n## Assistive Technology Categories\n\n### Screen Readers\n\n#### Background information\n\n- https://webaim.org/techniques/screenreader/\n- The BBC has some guides for testing: https://bbc.github.io/accessibility-news-and-you/accessibility-and-testing-with-assistive-technology\n\n#### Resources for common screen readers\n\n##### JAWS\n\n- Freedom Scientific has a detailed [JAWS HTML and ARIA support matrix](https://freedomscientific.github.io/VFO-standards-support/) and [a downloable version](https://www.freedomscientific.com/Downloads/JAWS) which may be used for up to 40 minutes without a license\n- https://bbc.github.io/accessibility-news-and-you/accessibility-and-testing-with-jaws\n\n##### Windows Narrator\n\n- https://support.microsoft.com/en-us/help/17173/windows-10-hear-text-read-aloud\n\n##### Apple (macOS and iOS) VoiceOver\n\n- https://help.apple.com/voiceover/info/guide/\n- https://bbc.github.io/accessibility-news-and-you/accessibility-and-testing-with-voiceover-ios\n- https://bbc.github.io/accessibility-news-and-you/accessibility-and-testing-with-voiceover-os\n"
},
{
"path": "docs/design-principles.md",
"content": "## Design Principles for System Features & Capabilities\n\nWe are collaborating across the Library of Congress to support this vision: _creating new pathways of engagement, scholarship, and serving the publicβwhile making history with people, staff, collections and systemsβby inviting public participation in a transformative project that improves discovery of the treasures of the Library of Congress._\n\nTwo principles guide these proposed features and the ways we can launch, manage, and sustain a crowdsourced transcription and tagging platform at the Library of Congress:\n\n**Trust** and **Approachability**\n\nThe system features and capabilities envisioned below will support a user-centered crowdsourcing initiative through functionality and a program of engagement.\n\n## **ENGAGE**\n\n### Participation by many audiences in the same place, at the same time\n\nWe have a unique opportunity to support many audiences including, but not limited to, a curious public, researchers, teachers, students, those seeking volunteer opportunities, and staff. This crowdsourcing platform presents a digital space in which audiences with different, but possibly overlapping interests, skill levels, and familiarity with the Library of Congress may gather together. This vision builds on the work Library of Congress staff have invested in crafting programs that serve tanging audiences based on their specific needs. We are likely to have a dynamic and constantly changing atmosphere of participants and collections; here are some essential approaches and features to accommodate many participants in the same place, at the same time.\n\n#### Lowest barrier to entry: No accounts necessary (but possible)\n\nTo ensure that every visitor can contribute and begin their Library of Congress crowdsourcing experience immediately, it would be best that accounts are not required to participate. Accounts may still be possible but not set as a prerequisite to contribute. Entries by "anonymous" users can be assessed separately or in a wider pool of contributions. As discussed below, it will be important to understand activity on the site; therefore, care should be taken to inform even anonymous users about the ways their behavior is captured. It is essential that these details are explained with clear language about the need for and use of that information.\n\n#### Accessibility\n\nLibrary of Congress audiences and visitors have a range of needs. This project should at all times meet the needs of all audiences. Examples of ways to create a welcoming and accommodating space include making it clear how to customize font sizes, interface adjustments, font styles and color choices, and plain language for instructions and project contexts. There are opportunities to invite participation in the system and use of the collections in many ways including reviewing and integrating voice to text capabilities, particularly for tagging. By meeting and thoughtfully surpassing compliance standards, the Library of Congress can create pathways for increased accessibility within the transcription workflow - such as the simple addition of a field to create alt text for the images. Captioned video content to support collections contexts, blog posts about discoveries and activity updates, and other featured content would also foreground that all audiences, even those who are not directly transcribing or tagging, are invited.\n\n#### Multiple entry points\n\nWe should anticipate visitors entering from many different paths: whether a visitor is routed through the homepage into a collection and then selects an individual page to transcribe and tag; or referred by a post on a social network; or uncovered in internet search engine results; or introduced via [https://loc.gov](https://loc.gov) or simply word of mouth; or returning to transcribe and tag again via their own account page. It should always be clear how a visitor would navigate to the instructions, to an overview of the crowdsourcing program, to move to one's account pages (if one has created an account), and to navigate "up" to the project or campaign page(s), if entering on an individual transcription or tagging page from persistent unique URL.\n\n#### Many Paths through Digital Collections\n\nHeterogeneous subjects, forms, stories, and tasks are key to sustaining a long-term and representative crowdsourcing project. It is likely that many visitors will follow the sequential presentation of images, just as they were reading a diary or letter, or following other documentation as it unfolds. However, other visitors may wish to explore pages across collections with minimal reorientation to the home or collection-level pages. Still other volunteers may wish to change tasks between transcribing and tagging, or performing review passes, while participating. Community managers may also wish to lead visitors to targeted activity. There are unique opportunities to use the tool with prepared and simultaneous programming around the collections and campaigns. It may be helpful to have "tracks" for the multiple audiences interacting in this dynamic space.\n\n#### Make Sense Quickly: Invitations to Contribute through Information Architecture\n\nUpon entering the platform, a visitor should understand what tasks are requested and the goals of the program. It should be easy to explore featured collections by tags identified by community managers and curators; it should also be possible to explore collections based on community or self-created tags. Some tag categories that might connect to audience interests include location, era, and event. It should also be easy to understand the historical contexts of the collections, including difficult topics and subject matter.\n\n#### Make it Easy to Participate\n\nFrom entry into the site, volunteers should feel welcomed and within only a few steps of getting started with contributing. Clear instructions that are accessible from many different points that incorporate illustrative examples; perhaps even instruction or participation support in varying formats, such as video. Helpful tips about material culture and practice, such as handwriting or paleography resources may further help. Technical features that support accuracy and ease of contribution include annotation tools and editors. Providing a mechanism for volunteers to elect to receive timely feedback on their efforts and provide similar to others are just some community management approaches that can support accuracy and ease of contribution.\n\n#### Serving Completed Data in Multiple Formats\n\nText and tags that are created by volunteers as they participate will be used to improve the ways collections can be found and connected. These forms of data should also be made available as project level JSON, CSV, and XML files, as well as a full corpus of completed transcription text. Ideally, these forms or data would be presented within the site on a "data" or research page. It should also be made clear the ways in which visitors might use the loc.gov interface to download individual images; and the loc.gov JSON API to download images and data from digital collections.\n\n#### Responsibly Share Code\n\nThoroughly documenting and sharing the design decisions that inform the crafting of the codebase will ensure that other libraries, cultural heritage organizations, and educational bodies can best decide if this tool is appropriate for their needs and matches the skills and resources they have to apply to a web-application based participatory project. We should appropriately document and make available the source code for the tool, as well as other technical considerations and design decisions. Furthermore, lessons learned in the process of developing and improving the underlying tool, as well as the program(s) of engagement made possible by the affordances of the tool, should be shared openly as supporting documentation and stand-alone considerations.\n\n#### Foregrounding Collaboration\n\nThere are many ways to signal respect for the contributions of volunteers while making the experience of participating most rewarding. One way is to maximize the extent of collaboration by better integrating opportunities to "positively compound" participation within the process of transcription, tagging, and review. Most existing tools use a blend of asynchronous transcription and either algorithmic review (matching) or volunteer review. Showcasing interpretation and activity by other volunteers can help participants craft a shared, agreed upon, and quality version of transcribed text. Displaying tags as categorization in the form of arranged and connected knowledge allows participants to crosswalk understanding, opening possibilities of new discovery and connection to the project mission and one another.\n\n## **UNDERSTAND**\n\n### Analyze activity & assess participant motivations to improve experiences\n\nIt will be imperative to responsibly gather, analyze, and share information about activity in the crowdsourcing platform. This information will be used to best communicate, improve upon existing capabilities, and extend future possibilities of the tool and program of engagement. This section discusses features and capabilities to support better experiences for all participants, from volunteers to staff.\n\n#### Privacy & Ethical Use of Data\n\nIt is imperative that we responsibly and carefully define the need to collect data - whether that is about activity, location, or other forms of information - and connect that explicitly to intended uses of that data to understand audience needs and improve service. However, it should not be required or possible to track users through their visits. Furthermore, it should be possible for users to have their accounts deleted and account information purged from the site, while retaining their anonymized contributions.\n\n#### No Accounts Necessary... but also creating an Account is possible\n\nAs noted above, it should be as easy as possible for visitors to become volunteers. However, creating an account within the system will allow participants to recall their activity, customize their experience, and fulfill needs for reporting should their motivations relate to formal volunteering or school assignments. We should explain to volunteers that creating an account affords additional possibilities including estimating participation. Accounts would also allow volunteers to experience other benefits: invitations to webinars with curators, alerts about new or related collections, and notifications of completion of projects or subjects of interest perhaps based on tags and other self-selection.\n\n#### Understanding for Community Management\n\nFor the health and future of the crowdsourcing initiative, gathering information about the efforts and communication of volunteers imperative. Presenting a clear snapshot of activity, recent discussion, possible roadblocks, and upcoming campaigns or communication would aid community management through quick assessment of the ecosystem's state. Easy access to "live data," rapidly gathering the pulse on high traffic projects or energetic discussion, and performance of ongoing campaigns are key needs for community managers. This scope of activity information would also be useful for community managers in discreetly and sparingly offering volunteers opportunities to re-engage with new collections.\n\n#### Motivations meet Behavior\n\nWe can help identify needed capabilities or features by surfacing the reasons people wish to participate and mapping patterns of activity in the system to these goals. The experience of participating in this project will be dynamic. Our visitors will not cleanly map onto single personas because their reasons for participating may change from visit to visit and over time. Common motivations include pursuing personal learning objectives, contributing to something greater and access to open knowledge, and to fulfill course or volunteer requirements. In 2014, researchers and project managers from the Zooniverse described the benefits of designing digital citizen science projects for participants with limited time and commitment; Eveleigh et al termed this approach "designing for dabblers." Crafting workflows that support bite-sized or small targeted tasks can meet the needs of this type of volunteer who may fit in participation in the crowdsourcing platform amid their other interests and activities. Other volunteers may seek more immersive experiences. Furthermore, students and other volunteers may have participation targets to meet during their time on site.\n\n#### Enabling Immersion\n\nFrom user interfaces to tracks, there are opportunities to design a site that enables immersive, deep engagement with stories in the collections, and to find flow in serialized tasks; yet also makes it possible to step out of the workflow quickly and with confidence that one's contributions will be retained and valued. Success may not always mean pace of transcription but rather the attainment of a visitor's goals, whether to learn, fulfill volunteering hours, or make a meaningful contribution to the Library of Congress.\n\n#### Volunteers with accounts should be able to understand their own activity\n\nParticipants in the crowdsourcing program who have created accounts should be able to quickly access, understand, filter or scope, print or download summaries of their activity. This information should be presented to them in a visually dynamic and customizable (or filtered) manner. It should also represent their cumulative activity as well as recent tasks, allow them to pick up where they left off, share or engage others with their own contributions, and perhaps suggest related content via tags or campaigns.\n\n#### Ability to gather feedback from participants about their needs\n\nResponses, questions, feedback, and other means of communication between participants and community managers will be essential to the health of the project. Beyond a feedback button or comment form, community managers will want to be able to assess participation activity data as feedback on complexities of the collections, spot barriers and drop offs, identify stickiness, and sight opportunities in content and behavior.\n\n#### Tell a Public Story by Displaying Activity\n\nPresenting information about system-wide activity, as well as individual activity, creates opportunities for shared understanding of the progress toward collective goals, as well as individualized approaches. This information can also be presented on an About or Homepage to quickly convey a sense of participation--via dynamic and perhaps interactive charts, torque maps, timelines, and more--and encourage visitors to join the activity.\n\n#### Reporting for Staff and Organization\n\nBeing able to provide reporting that integrates with or corresponds to staff workflows will help this project become more closely connected to regular activity in the organization. Information gathered about project performance might include the number of pages per collection, the number of volunteers that contributed, the number of views and engagement with the object in loc.gov/Project One, and details about visits including generalized time on site and return visits that engage with collections they've shared. It should be possible to query this data for custom reports, and include filters or other bounding options for specific time frames. Careful consideration should be taken around visitor information; never achieving tracking of users, and at all times generalized and disconnected from any cross-walkable search.\n\n## **CONNECT**\n\n### Manage & Match Collections to Tasks to Data and more\n\nThese activities focus on engaging audiences with Library of Congress collections. Simultaneously, we strive to design a series of tasks and support that result in data that may be applied to improve access to those Library collections. This section recommends tasks, design decisions, and processes to achieve these goals.\n\n#### Tasks: Transcription AND Tagging\n\nOf a range of possible tasks, we've identified transcription and tagging to be most relevant to our goals to improve search and identification within Library of Congress collections. These two tasks can be mapped to the digital content lifecycle in the description phase. These tasks are envisioned as a means of creating asset or page level text that can enhance discovery of and access to the Library of Congress digital collections.\n\n#### Transcription\n\nBased on the goals of engaging audiences and creating useful text for discovery, legibility, and access, the recommended transcription process is one that facilitates asynchronous but rapid access transcription; asks for minimal interaction with the collection asset image; situates the transcription window adjacent to the collection asset image; offers an adjustable and immersive presentation, if desired; "positively compounds" the efforts of participants; honors the time and contributions of participants; visually supports identification of possible errors or areas needing additional work; and connects to the context of the object in focus, whether smooth transition between pages before and after or to the catalog information and project description.\n\n#### Tagging\n\nThe goals of tagging are to classify or categorize the content in such a way as to make it discoverable for future use. It is recommended that the tagging feature be built to accommodate set(s) of collection level and platform-wide controlled vocabularies, as well as to accept crowd-generated tags. The former set of tags would allow the platform to leverage existing subject headings, known and popular loc.gov search terms, and other forms of metadata already associated with the collections. The latter tagging capability would allow volunteers to customize their experiences with collections once these tags were made available to search or surface content across the platform; it would also allow community managers to organize campaigns at the point of import and feature this content both on the homepage and highlighted throughout the experience of transcription and tagging. Finally, a visual coding of the tags would offer an opportunity for a subtle contextualization of staff and Library of Congress generated tags and those created by participants.\n\n#### Task Ecosystem: Tasks in the Workflow\n\nProjects and pilots including [Beyond Words](https://labs.loc.gov/experiments/beyond-words/) have demonstrated that the transcription and tagging tasks may successful be presented together in the crowdsourced workflow; however it should be possible for participants to elect to only complete one task, if they prefer. It is also possible that these tasks might be best presented in separate interfaces or as distinct workflows. The tasks should also be introduced to participants by describing the goals of the information that is being created, examples of how and where it will be discoverable after volunteers have contributed. Specifically, care should be taken to communicate the ways that the information created will be usedβwhether for search, for research, for publication and display, or to improve the features of the systemβas these details will shape the ways the tasks are undertaken by volunteers.\n\n#### Reaching Agreement\n\nAgreement around the final version of transcription text will be a process that should be achieved through displaying the work of volunteers and facilitating a process in which they can negotiate to reach consensus of the completeness and quality of the text. The collections we will ask the public to transcribe will be of varying format, even changing in format from asset image to asset image. Therefore, there are likely to many points at which interpretation will occur. As a result, workflows that allow participants to work together and create a shared understanding of the asset image (at object/collection level) are most likely to be successful when the opportunity to exchange interpretations are presented throughout the transcription and tagging processes. Features in support of negotiated consensus include an ability to see, mark, report, and/or correct errors in the text; to discuss an asset image or page in a forum; and generalized discussion at the collection level. Another way to permit negotiated consensus is to allow for a peer-review workflow.\n\n#### Complete the Cycle\n\nIt is imperative that the crowd-generated transcription text be returned and served with the [loc.gov](https://loc.gov) presentation of the object and at the asset image level. It may be possible to marry this information as metadata or perhaps as data supporting the collection or object. This urgency and responsibility in completing the cycle connects to a user-centered design; building trust with public participants by meeting motivations that relate to contributing to greater access to knowledge, while honoring their time and contribution.\n\n#### Connect Activity to Collections\n\nAt all times it should be possible to navigate from the crowdsourcing project, campaign, or transcription and tagging page to the source object. In the volunteer's account view, it should be possible to navigate from the project to which they contributed to the record of that collection in [loc.gov](https://loc.gov). Furthermore, there are opportunities to connect volunteers to the [Ask a Librarian](https://www.loc.gov/rr/askalib/) service from the crowdsourcing platform, such as in the discussion section, the project and collection pages, the homepage, and even in the footer.\n\n#### Managing Projects\n\nThese key features and approaches should be available for the long-term sustainability of the project:\n\n- Continuity with existing staff workflows, including work within CTS (content transfer system)\n- A pipeline for the queue of projects, as well as a monthly mechanism for proposing, identifying, and planning new collections for transcription including a recurring forum for nominating projects - perhaps even a sandbox or test space in which details of nominated collections may be stored or prepared\n- Collections management in bulk to support creating an extensive queue; storing offline in advance of campaign, collection, or other programming needs\n- Enabling Community Managers to post, launch and queue projects and campaigns via an administrative interface. Over time, this functionality can be expanded with a permissions-based self-service mechanism for curators and collections staff to identify, flag, or queue projects for transcription. This will likely require coherence with Library of Congress web services, Design & Development, as well as the CTS workflow\n\n#### Applying our APIs\n\nAs with other features recommended here, it would be best to seize and map to existing workflows and technologies. A goal for the platform should be to identify and deliver collections via [loc.gov](https://loc.gov) API to become transcription projects. Furthermore, allowing the transcription results to be queried via the [loc.gov JSON API](https://libraryofcongress.github.io/data-exploration/) once associated with the source images or record would be valuable.\n\n#### Transparent Development\n\nThe greatest opportunity exists to develop this tool and platform in the open, as open source with appropriate licensing. Documenting development in a repository like GitHub is advisable.\n\n#### Identify, Articulate, Acknowledge Content Created by Volunteers\n\nIdentify content created by participants within loc.gov as volunteer-created. Allow volunteers with an account to retain a pointer to the work they created, with a persistent URL or URI. Actively acknowledge the contributions of volunteers in publications, presentations, social media, and communications outreach. License for content created in the transcription and tagging workflow should be public domain or CC0; it should also be explicitly stated and incorporated as metadata when presented as a dataset and in loc.gov. These contexts should be clearly communicated to volunteers contributing the content as well as researchers engaging with the content.\n\n#### Data Available for Download\n\nTo compliment research and exploration at the object and collection level, providing transcription and tagging results as a bulk data set would allow researchers in a range of disciplines to explore patterns and connections across collections. The transcription text and tags should be licensed as openly as possible and volunteers should be kept informed of the ways their efforts support discovery in the Library of Congress systems, as well as the role their work plays in other scholarly inquiry.\n\n#### Navigating options to get started\n\nVisitors and volunteers should be able to navigate the available collections in several ways. For example, presenting available projects as a list or set of tiles and incorporating filtering and sorting capability would allow volunteers to swiftly make sense of the active opportunities. Incorporating existing metadata offers other ways to orient volunteers to opportunities. For example, offering selection based on the object time period or era, location metadata, or subject heading. Finally, as participation increases, it may be possible to estimate time to complete a project based on participation data. This approach serves the needs of visitors who wish to dedicate a specific amount of time to their visit to the project.\n\n### **GROW**\n\n### Sustaining and Improving through Workflows and Spaces of Participation\n\nOnce the system has been designed, developed, and launched and once visitors become volunteers, there are endless opportunities to improve and sustain the platform and the program based on the ways people use it, their needs and challenges, and information gathered in the process of participation.\n\n#### Show the Work!\n\nDisplaying the work of others in the system allows participants, whether new or seasoned, to more quickly understand that others are contributing to shared goals. The visibility of content & contribution also signals active collaboration. Furthermore, visual indications of collaboration, whether intended or even in conflict, can be reflected in an interface that shows the efforts of editing. Allowing content to be editable, then marking out those editsβwith font, size or other indicatorβallows individuals to better understand who and what has come before them on the page.\n\n#### Create an Atmosphere\n\nCreating a shared understanding of expectations around behavior and communication can help shape expectations of respect and civility in this space. Inviting participants to acknowledge and commit to a Code of Conduct provides them the opportunity to reflect on how they will engage with others, as well as offers them support for framing communication they receive. Examples of public community code of conduct include the Coral Project and Airbnb; the former articulates acceptable and unacceptable behavior and expectations, while the latter asks customers (guests) to sign a code of conduct pledge as part of the booking process.\n\n#### Clear Examples\n\nAs participants create transcription and tagged content and more collections are added, plenty of examples to best represent what is needed when encountering decision-making moments will emerge. In the interim, creating clear and annotated examples may be useful to those who are just getting started with transcription and tagging.\n\n#### Collate and Connect Extraneous Knowledge\n\nAs volunteers participate in the crowdsourcing initiative, they will acquire, recall, and perhaps seek to share information and knowledge with others; perhaps to be helpful to other volunteers and, at other times, as an expression of their interest and engagement with Library of Congress collections. They may wish to share examples and further non-Library of Congress resources. A discussion board or forum can help volunteers achieve their motivations of learning, contributing to wider knowledge (a greater good), and to build community or a sense of shared purpose through discussion. It may also be useful to create an asset level discussion for specific or nuanced questions about the asset image or page that is receiving transcription and tagging activity. Impressive examples of discussion spaces in crowdsourcing projects include the Zooniverse talk pages and the Discourse implementation in use by In the Spotlight at the British Library.\n\n#### Sustain Interest and Increase the Capability of Participants\n\nAs described above, visitors may have many different motivations when they first arrive at the crowdsourcing platform; they may also have different motivations each time they arrive on site. Their movement to becoming volunteers may be catalyzed by offering a range of tasks, heterogeneous collections, and oscillating levels of complexity. Furthermore, the system may be built with tracks that prompt a participant to continue in the next step, or asks them to contribute to a new task. It is also possible to blend these approaches and particularly helpful if these are cyclical or distributed opportunities - either naturally based on variety in the collections or designed by community management approaches. Furthermore, offering opportunities to problem-solve, support other volunteers in other roles, level up in transcription and tagging tasks, or self-select for more complex (or pilot) tasks can create a dynamic experience that is rewarding to volunteers and the program ecosystem.\n\n#### Build Knowledge for Outcomes\n\nIt is recommended that Community Managers and staff with collections expertise work together closely to build programming and engagement in relation to the transcription and tagging tasks. This programming could take the shape of essays, webinars, edited video content, chats hosted in discussion pages or social networks, and in-person events. In these activities, persistent URLs for the page, collection, and campaign (or collection of tags) would be required. In the previous section, recommendations were shared for connecting activity to collections. It is also possible to connect collections to catalyze activity; for example, a prompt or pathway from the collection or object Project One page to the crowdsourcing platform.\n\n#### Practice\n\nOne way to encourage activity is to offer a low risk point of entry in which a volunteer might practice and keep notes, such as in a sandbox. This space may also make it possible for a volunteer to elect to receive feedback that can increase their confidence, coherence and consistency in transcription and tagging, and connection between their goals and those of the program. Another means of training could be a practice page, potentially offered after a volunteer has made a few contributions or after they have created an account.\n\n#### Piloting\n\nThe Library of Congress Labs team is well-situated to partner with staff in the Office of the Chief Information Officer and Library Services to continue to run crowdsourcing experiments focused on tasks, cataloging and metadata, and machine learning. Examples include working with Optical Character Recognition (OCR) and Handwritten Text Recognition (HTR) outputs to assess quality of transcription & assistance with tagging; parsing task workflows; repeated passes on the same material for thematic, semantic, or other tagging; named entity recognition and more. Furthermore, there are opportunities to integrate other collections and materials into a crowdsourced workflow including audio-visual and time-based media objects. Creating an extensible tool or system into which features that emerge from experiments could be added would support the evolution of the project, as well as create opportunities for volunteers to grow their skills and knowledge.\n"
},
{
"path": "docs/for-developers.md",
"content": "# For Developers\n\n## Prerequisites\n\nThis application can run on a single Docker host using docker-compose.\n(recommended for development environments). See the [development Readme](../development/README.md) for more information. Note that the instructions below assume you'll be developing on your host rather than in a container. The development Readme provides instructions on performing development in a purely containerized environment, without installing any dependencies (outside of git and your container tool of choice) on the host.\n\nFor production, see the\n[cloudformation](https://github.com/LibraryOfCongress/concordia/tree/master/cloudformation) directory for AWS Elastic Container Service\nstack templates.\n\n## Running Concordia\n\n### Docker Compose\n\n```bash\ngit clone https://github.com/LibraryOfCongress/concordia.git\n```\n\nIf you're intending to edit static resources, templates, etc. and would like to\nenable Django's DEBUG mode ensure that your environment has `DEBUG=true` set\nbefore you run `docker-compose up` for the `app` container. The easiest way to\ndo this permanently is to add it to the `.env` file:\n\n```bash\necho DEBUG=true >> .env\n```\n\n##### Install the application virtual environment\n\nThese steps only need to be performed the first time you setup a fresh\nvirtualenv environment:\n\n1. Ensure that you have the necessary C library dependencies available:\n\n - `libmemcached`\n - `postgresql`\n - `node` & `npm` for the front-end tools\n\n1. Ensure that you have Python 3.8 or later installed\n\n1. Install [pipenv](https://docs.pipenv.org/) either using a tool like\n [Homebrew](https://brew.sh) (`brew install pipenv`) or using `pip`:\n\n ```bash\n pip3 install pipenv\n ```\n\n1. If you encounter errors installing psycopg, you may need to set LDFLAGS in your environment variables.\n\n1. Let Pipenv create the virtual environment and install all of the packages,\n including our developer tools:\n\n ```bash\n pipenv install --dev\n ```\n\n n.b. if `libmemcached` is installed using Homebrew you will need to [set the CFLAGS long enough to build it](https://stackoverflow.com/questions/14803310/error-when-install-pylibmc-using-pip#comment94853072_19432949):\n\n ```bash\n CFLAGS=$(pkg-config --cflags libmemcached) LDFLAGS=$(pkg-config --libs libmemcached) pipenv install --dev\n ```\n\n Once it has been installed you will not need to repeat this process unless\n you upgrade the version of libmemcached or Python installed on your system.\n\n1. Configure the Django settings module in the `.env` file which Pipenv will use\n to automatically populate the environment for every command it runs:\n\n ```bash\n echo DJANGO_SETTINGS_MODULE=\"concordia.settings_dev\" >> .env\n ```\n\n You can use this to set any other values you want to customize, such as\n `POSTGRESQL_PW` or `POSTGRESQL_HOST`.\n\n n.b to allow a local server to connect to the dockerized db set `POSTGRESQL_PORT=54323` - the db containers external postgres port.\n\n1. Make sure that [redis](https://redis.io/docs/getting-started/) is installed and\n running.\n\n1. Configure Turnstile in your `.env` file. Unless specifically testing Turnstile,\n you'll probably want the following settings:\n\n ```bash\n echo TURNSTILE_SITEKEY=1x00000000000000000000BB >> .env\n echo TURNSTILE_SECRET=1x0000000000000000000000000000000AA >> .env\n ```\n\n Those two settings ensure all Turnstile tests pass. See [Turnstile Testing](https://developers.cloudflare.com/turnstile/troubleshooting/testing/) for other options.\n\n### Local Development Environment\n\nYou will likely want to run the Django development server on your localhost\ninstead of within a Docker container if you are working on the backend. This is\nbest done using the same `pipenv`-based toolchain as the Docker deployments:\n\n#### Python Dependencies\n\nPython dependencies and virtual environment creation are handled by\n[pipenv](https://docs.pipenv.org/).\n\nIf you want to add a new Python package requirement to the application\nenvironment, it must be added to the Pipfile and the Pipfile.lock file.\nThis can be done with the command:\n\n```bash\npipenv install \n```\n\nIf the dependency you are installing is only of use for developers, mark it as\nsuch using `--dev` so it will not be deployed to servers β for example:\n\n```bash\npipenv install --dev django-debug-toolbar\n```\n\nBoth the `Pipfile` and the `Pipfile.lock` files must be committed to the source\ncode repository any time you change them to ensure that all testing uses the\nsame package versions which you used during development.\n\n#### Launching the environnment\n\nIn order to successfully launch the environment, the environment variables\n`POSTGRESQL_PW` and `DJANGO_SETTINGS_MODULE` must be set. `POSTGRESQL_PW`\nmay be set to any value (which will become the database password for the\nenvironment), but `DJANGO_SETTINGS_MODULE` should be set to\n`concordia.settings_dev` to use the development settings file.\n\n```bash\nexport POSTGRESQL_PW=password\nexport DJANGO_SETTINGS_MODULE=concordia.settings_dev\n```\n\n```bash\ncd concordia\ndocker-compose up\n```\n\nBrowse to [localhost](http://localhost)\n\n#### Setting up a local development server\n\n##### See section - [Ensuring your work follows the Library's coding standards](https://github.com/LibraryOfCongress/concordia/blob/master/docs/how-we-work.md#ensuring-your-work-follows-the-librarys-coding-standards) in How We Work\n\n##### Start the support services\n\nInstead of doing `docker-compose up` as above, instead start everything except the app:\n\n```bash\ndocker-compose up -d db redis importer celerybeat\n```\n\nThis will run the database in a container to ensure that it always matches the\nexpected version and configuration. If you want to reset the database, simply\ndelete the local container so it will be rebuilt the next time you run\n`docker-compose up`: `docker-compose rm --stop db`.\n\n##### Install front end\n\n1. Install Node 20. If you're on MacOS, you can install it using brew:\n\n ```bash\n brew install node@12\n ```\n\n1. Use NPM to install our development tools:\n\n ```bash\n npm install\n ```\n\n1. In another terminal, start Vite to watch for changes to the SCSS files and\n compile them to CSS, and changes in the bundled, hased and compressed js files:\n\n ```bash\n npx vite\n ```\n\n If you only want to build, bundle, and compress them a single time without live updates:\n\n ```bash\n npx vite build\n ```\n\n1) You may need to manually create a logs directory.\n\n ```bash\n mkdir logs\n ```\n\n1) Collect Django static files:\n\n ```bash\n pipenv run ./manage.py collectstatic --no-post-process\n ```\n\n##### Start the application server\n\n1. Apply any database migrations:\n\n ```bash\n pipenv run ./manage.py migrate\n ```\n\n1. Start the development server:\n\n ```bash\n pipenv run ./manage.py runserver\n ```\n\n#### Running the unit tests\n\nUse the `settings_local_test` Django settings in your environment. Your `.env` file should look something like:\n\n```bash\nPOSTGRESQL_PW=password\nDJANGO_SETTINGS_MODULE=concordia.settings_local_test\n```\n\nBring up the docker database and redis servers:\n\n```bash\ndocker-compose up -d db redis\n```\n\nThen execute the tests:\n\n```bash\npipenv run ./manage.py test\n```\n\n#### Import Data\n\nOnce the database, redis service, importer and the application\nare running, you're ready to import data.\nFirst, [create a Django admin user](https://docs.djangoproject.com/en/2.1/intro/tutorial02/#creating-an-admin-user)\nand log in as that user.\nThen, go to the Admin area (under Account) and click \"Bulk Import Items\".\nUpload a spreadsheet populated according to the instructions. Once all the import\njobs are complete, publish the Campaigns, Projects, Items and Assets that you\nwish to make available.\n\n#### Data Model Graph\n\nTo generate a model graph, make sure that you have [GraphViz](https://graphviz.org/doc/info/command.html) installed (e.g.\n`brew install graphviz` or `apt-get install graphviz`) and use the\n[django-extensions `graph_models`](https://django-extensions.readthedocs.io/en/latest/graph_models.html) command:\n\n```bash\ndot -Tsvg <(pipenv run ./manage.py graph_models concordia importer) -o concordia.svg\n```\n\n## Other Front-End Tools\n\n### Public-facing URLs\n\n1. If you need a list of public-facing URLs for testing, there's a management\n command which may be helpful:\n\n ```bash\n pipenv run ./manage.py print_frontend_test_urls\n ```\n\n### Accessibility testing using aXe\n\nAutomated tools such as [aXe](https://www.deque.com/axe/) are useful for\ncatching low-hanging fruit and regressions. You run aXe against a development\nserver by giving it one or more URLs:\n\n```bash\nnpx axe-cli --show-errors http://localhost:8000/\npipenv run ./manage.py print_frontend_test_urls | xargs npx axe-cli --show-errors\n```\n\n### Static Image Compression\n\nWhen you update any of the files under `concordia/static/img`, please use an\noptimizer such as [ImageOptim](https://imageoptim.com) or [Caesium](https://caesium.app/)\nto **losslessly** compress JPEG, PNG, SVG, etc. files.\n\n```bash\nbrew cask install imageoptim\nopen -a ImageOptim concordia/static/img/\n```\n"
},
{
"path": "docs/how-we-work.md",
"content": "# How We Work\n\n## Principles\n\nOur basic principles are:\n\n- We produce open source software, shared in repositories where it may be inspected by the public who places their trust in it and copied for use by other agencies or institutions.\n- We adhere to the basic practices of agile software development, using the Scrum development framework.\n- We practice human-centered design. Everything that we produce is highly accessible, per [WCAG 2.1](https://www.w3.org/TR/WCAG21/).\n- Finally, we believe in having the relevant decision-makers at the table during all meetings, to maximize efficiency and maintain momentum.\n\n## Product Team\n\nThis is a cross functional product team for Concordia made up of members across the Library who are working together. This product team will be comprised of the following roles:\n\n- Product owner\n- Product manager (Scrum master)\n- Technical lead\n- User Experience designer\n- Developers (Front-end, Back-end, Full-stack)\n- QA Tester\n- Community Managers (content writers, administrators)\n\nThis team participates in stand ups, product alignment, backlog grooming and retrospectives in service of prioritizing, defining and delivering value to the department and the public it serves.\n\n## Sprint Organization and Meetings\n\nEach sprint is three weeks long. We have a sprint kick off the first day of the new sprint. There are five basic meeting rhythms:\n\n- **Backlog grooming and Sprint Planning, every 2 weeks**\n - Structure: tickets in the backlog are sorted by priority, the team adds acceptance criteria, estimates size, and assigns the tasks to a team member.\n- **Demo and retrospectives, every 2 weeks**\n - At the end of each sprint, Developers or content writers demo completed work in the sprint for the larger library stakeholders. During demo, we will confirm if the user acceptance criteria is met and moved to be tested. Following the demo, the team will go through a retrospective. These are held back-to-back, on the same day\n\n## Definition of Done\n\nSo that we can work more efficiently and be confident in the quality of the work we are delivering, we have a clear definition of what it means for a user story to be done, or production-ready.\n\n**For delivering a user story to the product team:**\n\n- Story needs to be written in a way that is clear from both a development and a testing standpoint. Stories will need to be reviewed by the product team during creation.\n- Acceptance criteria should include the relevant tests needed (unit, security, performance, acceptance, etc)\n- Acceptance criteria should include the objective of the story, for use in approval by PO or tech team or both\n- The delivered functionality should match the acceptance criteria of the user story\n\n**for product team to accept the user story and ship it:**\n\n- The functionality meets the acceptance criteria\n- The product team has verified the functionality in staging\n- All tests must pass in the the stage environment (unit, integration, feature)\n- The delivered functionality should be 508 compliant\n- Security requirements must be met\n- All documentation must be up to date (diagrams, training documentation, API documentation, help text, etc)\n- The delivered functionality should be compatible with the latest versions of Firefox, Chrome and Safari\n\n## Processes\n\n### Testing Strategy\n\nWe practice testing at three levels: unit tests, integration tests, and feature tests. For details about how we create and maintain unit, integration and feature tests.\n\n- Unit - Unit tests must be created for all new code, during the sprint in which the code is written, with coverage of at least 90%.\n- Integration - Code must include tests that verify that interfaces are functioning as designed.\n- Feature - New features must have functional definitions of the thing that they are to perform, and a description of human-performable actions to verify that they perform that thing.\n\n#### Testing new code\n\nEach ticket will include acceptance tests, especially for new user facing functionality. When the developer signals that the ticket is ready for test, they will move it to the `Needs test` column and pings the tester in the comment of the ticket.\n\nTesters will identify issues - HIGH, LOW, or NONE. Here are the criteria for each of the levels:\n\n- FAIL: Does not meet the acceptace criteria and can not complete the acceptance test.\n- PASS: Does meet both acceptace criteria and and acceptance tests. When a ticket passes but there are noticable opportunities for improvements or enhancements, close the ticket and create a new ticket and add to the backlog.\n\nIf all user acceptance criteria has been met, the ticket will be closed and moved to Done column.\n\nFinal step: Technical lead will create a release with all tickets that are done.\n\n**How to provide feedback**\n\nIf feature testers find issues to address:\n\n- FAIL: comment and @ the developer in the feature ticket\n- Enhancements or Improvements: open a new FEATURE ticket. This ticket will be added to the backlog and up for priortization in alignment meeting and backlog grooming. Link to related ticket by adding the issue #.\n\nIf a FAIL issue needs to be addressed, tester should expect to be available to respond to developers questions and retest until acceptance criteria is met.\n\n### Ticket Movement in a Sprint\n\n**For a new feature ticket:**\n\n1. Ticket is generated as an issue and placed in the backlog\n2. Product owner will place priority tickets in a sprint\n3. Product Manager will ensure all acceptance criteria has been articulated and assign developer\n4. Developer will move to In Progress when ticket is being worked on\n5. When Developer has completed initial code will move code into crowd-test.loc.gov\n6. Developer moves ticket into test assigned Product Owner to review feature needs\n7. Product Owner test the feature and test if Acceptance Criteria is met, provides feedback to devs if needed\n8. Product Owner approves feature then assigns to QA\n9. QA will affirm AC and test broader functionality and accessibility. Will pass/fail ticket. Outcome of testing will be written in comments.\n - If pass, QA will close ticket and move to Done.\n - If fail, QA will move ticket back into In Progress and assigned back to developer\n10. If additional issues are found in testing that are not related to the feature of functionality, new tickets will be written by QA and added to the backlog. Ticket will be assigned to PO for further ticket development and grooming.\n\n**For system wide upgrades**\n\n1. Ticket is generated as an issue and placed in the backlog in GitHub\n2. Technical Lead will place priority tickets in a sprint\n3. Product Manager will ensure all acceptance criteria has been articualted and assign developer\n4. Developer will move to In Progress when ticket is being worked on\n5. Unit testing incorporated into the code pipeline\n - Manually run unit tests\n - CI/CD integration\n6. Move ticket into test and assign peer review testing by developer\n - If fail, developer provides feedback, moves ticket back to In Progress and assigns to the original dev\n - If pass, assign to QA for regression testing\n7. Tester will copy the [Regression Testing Checklist](https://staff.loc.gov/wikis/x/UomCBQ) as a comment in the ticket\n8. Tester will go through all the functionality described in the checklist\n - If fail, provides feedback in the comments, moves ticket back to In Progress and assigns to dev\n - If pass, checks boxes to show that all main functionality are working and moves ticket to done\n\n### Branch strategy and Pull Request Process\n\n#### Git branching strategy\n\nWe have two long-lived git branches, `master` and `release`.\n\nThe `master` branch continuously deploys to our development environment.\n\nThe `release` branch continuously deploys to our staging environment.\nOur development and staging environments are on AWS and only accessible through the Library's network.\n\n##### Starting new work\n\nWhen someone begins new work, they cut a new branch from `master` and name it after their work, perhaps `feature1`. New changes are pushed to the feature branch origin.\n\n##### Merging to `master`\n\nWhen new work is complete, we set up a Pull Request (PR) from `feature1` to `master`. Discussion about, and approval of changes by either the Technical Lead, Product Owner or both happens in the PR interface in GitHub.\n\nOnce this new work is approved we merge the code, which closes the PR.\nFrom here, our CI pipeline will build the new changes on the `master` branch. Next, our CD pipeline will deploy the new work to our development environment.\n\n##### Merging to `release`\n\nOnce the development work on a sprint is completed, we set up a PR from `master` to `release`.\n\nThis constitutes a new release candidate. Any last-minute discussion, as well as approval happens in the PR interface. Once approved by the Technical Lead, Product Owner or both and merged, CI runs for `release` branch to the staging environment.\n\n##### Tagging and deploying to production\n\nWhen the `release` branch has been fully tested in the staging environment, we create a GitHub release with a tag on the `release` branch.\n\nEither trigger a Jenkins build manually or wait for continuous integration for the `release` branch to kick in. This will build a cleanly tagged versioned release candidate and upload the docker images to Amazon Elastic Container Registry.\n\nTo deploy to production, create a new task revision for the `concordia-prod` task which updates the version numbers of the docker containers to match the recently built cleanly tagged release candidate. Update the production service to use the new task definition revision. Monitor the health check endpoint to ensure the service is updated to the proper version.\n\n##### Patching production mid-sprint\n\nIf a problem is identified in production that needs a quick fix, we code the fix to production in a new branch cut from `release`, maybe called `prod_fix`. We set up a PR against `release` for review and discussion.\n\nAny QA or manual testing will take place in the staging environment deployed from the `release` branch. Once the release is tagged and deployed to production, we have to bring those new changes in release back into master. We use rebase again: `git rebase master release`.\n\n### Code quality and review process\n\nCode reviews are about keeping code clean and limiting technical debt. We will look for things that increase technical debt or create an environment where technical debt can be introduced easily later. Each pull request will be reviewed by the technical lead or assigned reviewer. As a reviewer, they will look closely for untested code, if there are tests that they are testing what they're supposed to, that they are following the Library's code standards.\n\n### Ensuring your work follows the Library's coding standards\n\nThe project extends the standard Django settings model for project configuration and the Django test framework for unit tests.\n\n#### Configuring your virtual env\n\nThe easiest way to install the site is using [Pipenv](https://pipenv.readthedocs.io/) to manage the virtual environment and install dependencies.\n\n#### Configure your local checkout with code-quality hooks\n\n1. Install [pre-commit](https://pre-commit.com/)\n1. Run `pre-commit install`\n\nNow every time you make a commit in Git the various tools listed in the next\nsection will automatically run and report any problems.\n\nn.b. Each time you check out a new copy of this Git repository, run `pre-commit install`.\n\n#### Configure your editor with helpful tools:\n\n[setup.cfg](https://github.com/LibraryOfCongress/concordia/blob/master/setup.cfg) contains configuration for [pycodestyle](https://pypi.org/project/pycodestyle/), [isort](https://pypi.org/project/isort/) and [flake8](https://pypi.org/project/flake8/).\n\nConfigure your editor to run black and isort on each file at save time.\n\n1. Install [black](https://pypi.org/project/black/) and integrate it with your editor of choice.\n2. Run [flake8](http://flake8.pycqa.org/en/latest/) to ensure you don't increase the warning count or introduce errors with your commits.\n3. This project uses [EditorConfig](https://editorconfig.org) for code consistency.\n\nIf you can't modify your editor, here is how to run the code quality\ntools manually:\n\n```\n $ black .\n $ isort --recursive\n```\n\nBlack should be run prior to isort. It's recommended to commit your code\nbefore running black, after running black, and after running isort so\nthe changes from each step are visible.\n\n## Tools we use\n\n- GitHub - We use our GitHub organization for storing both software and collaboratively-maintained text.\n- Slack - We use the Slack for communication that falls outside of the structure of Jira or GitHub, but that doesnβt rise to the level of email, or for communication that itβs helpful for everybody else to be able to observe.\n- WebEx - We use WebEx for video conferencing in all our meetings\n"
},
{
"path": "entrypoint.sh",
"content": "#!/bin/bash\n\nset -e -u # Exit immediately for unhandled errors or undefined variables\n\nmkdir -p /app/logs\ntouch /app/logs/concordia.log\n\necho \"Running makemigrations\"\n./manage.py makemigrations --merge --noinput\n\necho \"Running migrations\"\n./manage.py migrate\n\necho \"Ensuring our base configuration is present in the database\"\n./manage.py ensure_initial_site_configuration\n\nif [ -v SENTRY_BACKEND_DSN ]; then\n echo \"Testing Sentry configuration\"\n echo \"from sentry_sdk import capture_message;capture_message('This is a test event');\" | ./manage.py shell\nfi\n\necho \"Running Django ASGI server\"\ndaphne -b 0.0.0.0 -p 80 concordia.asgi:application\n"
},
{
"path": "exporter/__init__.py",
"content": ""
},
{
"path": "exporter/admin.py",
"content": "# Register your models here.\n"
},
{
"path": "exporter/apps.py",
"content": "from django.apps import AppConfig\n\n\nclass ExporterConfig(AppConfig):\n name = \"exporter\"\n"
},
{
"path": "exporter/exceptions.py",
"content": "from typing import List, Tuple\n\n\nclass UnacceptableCharacterError(ValueError):\n \"\"\"\n Raised when unacceptable characters are discovered in text to be exported.\n\n Each violation is stored so that callers can inspect which line / column held\n the character.\n\n Args:\n violations: A list of `(line, column, character)` triples representing\n every disallowed character found. Line and column numbers are both\n **1-based** so they can be reported directly to users.\n \"\"\"\n\n def __init__(self, violations: List[Tuple[int, int, str]]):\n self.violations: List[Tuple[int, int, str]] = violations\n details = \", \".join(\n f\"line {ln} col {col} -> {ch!r}\" for ln, col, ch in violations\n )\n super().__init__(f\"Unacceptable characters found: {details}\")\n"
},
{
"path": "exporter/migrations/__init__.py",
"content": ""
},
{
"path": "exporter/models.py",
"content": "# Create your models here.\n"
},
{
"path": "exporter/tabular_export/admin.py",
"content": "# encoding: utf-8\n\"\"\"\nHelpers for exporting Django admin querysets as Excel or CSV files.\n\nUsage in a ModelAdmin:\n\n actions = (export_to_excel_action, export_to_csv_action)\n\nThese actions take the current queryset and export it using the same field\nselection you would get from `values()` by default. The download filename is\nderived from the `ModelAdmin.model._meta.verbose_name_plural` unless a custom\nfilename is passed.\n\nThese helpers are adapted from the original django-tabular-export implementation:\nhttps://github.com/LibraryOfCongress/django-tabular-export/blob/master/tabular_export/admin.py\n\"\"\"\n\nfrom functools import wraps\nfrom typing import Any, Callable, Iterable\n\nfrom django.contrib.admin import ModelAdmin\nfrom django.db.models import QuerySet\nfrom django.http import HttpRequest, HttpResponse\nfrom django.utils.encoding import force_str as force_text\nfrom django.utils.translation import gettext_lazy as _\n\nfrom .core import (\n export_to_csv_response,\n export_to_excel_response,\n flatten_queryset,\n)\n\n\ndef ensure_filename(suffix: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]:\n \"\"\"\n Decorator factory to ensure a default filename for export admin actions.\n\n If the wrapped action is called with ``filename=None``, the filename is\n built from ``modeladmin.model._meta.verbose_name_plural`` plus the given\n suffix.\n\n Args:\n suffix (str): File extension to append (for example, ``\"csv\"`` or\n ``\"xlsx\"``).\n\n Returns:\n Callable[[Callable[..., Any]], Callable[..., Any]]: A decorator that\n wraps an admin action and injects a default filename when needed.\n \"\"\"\n\n def outer(f: Callable[..., Any]) -> Callable[..., Any]:\n @wraps(f)\n def inner(\n modeladmin: ModelAdmin,\n request: HttpRequest,\n queryset: QuerySet[Any],\n filename: str | None = None,\n *args: Any,\n **kwargs: Any,\n ) -> HttpResponse:\n if filename is None:\n filename = \"%s.%s\" % (\n force_text(modeladmin.model._meta.verbose_name_plural),\n suffix,\n )\n return f(\n modeladmin,\n request,\n queryset,\n *args,\n filename=filename,\n **kwargs,\n )\n\n return inner\n\n return outer\n\n\n@ensure_filename(\"xlsx\")\ndef export_to_excel_action(\n modeladmin: ModelAdmin,\n request: HttpRequest,\n queryset: QuerySet[Any],\n filename: str | None = None,\n field_names: Iterable[str] | None = None,\n extra_verbose_names: dict[str, str] | None = None,\n) -> HttpResponse:\n \"\"\"\n Django admin action that exports selected records as an Excel XLSX download.\n\n The queryset is first flattened via :func:`flatten_queryset`, optionally\n restricted to the provided ``field_names`` and ``extra_verbose_names``,\n then returned as an XLSX file response.\n\n Args:\n modeladmin (ModelAdmin): The Django admin class that owns this action.\n request (HttpRequest): The current admin request.\n queryset (QuerySet[Any]): The selected objects to export.\n filename (str | None): Optional download filename. When omitted, a\n name is generated from the model's ``verbose_name_plural`` and the\n ``\"xlsx\"`` suffix.\n field_names (Iterable[str] | None): Optional iterable of field names to\n include in the export. When omitted, the default flattening logic\n is used.\n extra_verbose_names (dict[str, str] | None): Optional mapping of field\n names to custom column headers.\n\n Returns:\n HttpResponse: A response containing the XLSX file.\n \"\"\"\n headers, rows = flatten_queryset(\n queryset,\n field_names=field_names,\n extra_verbose_names=extra_verbose_names,\n )\n return export_to_excel_response(filename, headers, rows)\n\n\nexport_to_excel_action.short_description = _(\"Export to Excel\")\n\n\n@ensure_filename(\"csv\")\ndef export_to_csv_action(\n modeladmin: ModelAdmin,\n request: HttpRequest,\n queryset: QuerySet[Any],\n filename: str | None = None,\n field_names: Iterable[str] | None = None,\n extra_verbose_names: dict[str, str] | None = None,\n) -> HttpResponse:\n \"\"\"\n Django admin action that exports selected records as a CSV download.\n\n The queryset is first flattened via :func:`flatten_queryset`, optionally\n restricted to the provided ``field_names`` and ``extra_verbose_names``,\n then returned as a CSV file response.\n\n Args:\n modeladmin (ModelAdmin): The Django admin class that owns this action.\n request (HttpRequest): The current admin request.\n queryset (QuerySet[Any]): The selected objects to export.\n filename (str | None): Optional download filename. When omitted, a\n name is generated from the model's ``verbose_name_plural`` and the\n ``\"csv\"`` suffix.\n field_names (Iterable[str] | None): Optional iterable of field names to\n include in the export. When omitted, the default flattening logic\n is used.\n extra_verbose_names (dict[str, str] | None): Optional mapping of field\n names to custom column headers.\n\n Returns:\n HttpResponse: A response containing the CSV file.\n \"\"\"\n headers, rows = flatten_queryset(\n queryset,\n field_names=field_names,\n extra_verbose_names=extra_verbose_names,\n )\n return export_to_csv_response(filename, headers, rows)\n\n\nexport_to_csv_action.short_description = _(\"Export to CSV\")\n"
},
{
"path": "exporter/tabular_export/core.py",
"content": "# encoding: utf-8\n\"\"\"Exports to tabular (2D) formats\n\nThis module contains functions which take (headers, rows) pairs and return\nHttpResponses with either XLSX or CSV downloads\n\nThe ``export_to_FORMAT_response`` functions accept a ``filename``, and\n``headers`` and ``rows``. This allows full control over the data using\nnon-database data-sources, the Django ORM's various aggregations and\noptimization methods, generators for large responses, control over the\ncolumn names, or post-processing using methods like ``get_FOO_display()``\nto format the data for display.\n\nThe ``flatten_queryset`` utility used to generate lists from QuerySets\nintentionally does not attempt to handle foreign-key fields to avoid\nperformance issues. If you need to include such data, prepare it in advance\nusing whatever optimizations are possible and pass the data in directly.\n\nIf your Django settings module sets ``TABULAR_RESPONSE_DEBUG`` to ``True``\nthe data will be dumped as an HTML table and will not be delivered as a\ndownload.\n\nOriginally from\nhttps://github.com/LibraryOfCongress/django-tabular-export/blob/master/tabular_export/core.py\n\"\"\"\n\nimport csv\nimport datetime\nfrom functools import wraps\nfrom itertools import chain\nfrom typing import Any, Callable, Iterable, Mapping, Sequence\nfrom urllib.parse import quote\n\nimport xlsxwriter\nfrom django.conf import settings\nfrom django.db.models import QuerySet\nfrom django.http import HttpResponse, StreamingHttpResponse\nfrom django.utils.encoding import force_str\n\nResponseType = HttpResponse | StreamingHttpResponse\n\n\ndef get_field_names_from_queryset(qs: QuerySet[Any]) -> list[str]:\n \"\"\"\n Derive field names from a queryset, including extra and aggregate columns.\n\n The queryset is first coerced to a ``values()`` queryset so that extra\n selects and annotations appear with the same names Django would use for\n ``values()`` results.\n\n Args:\n qs: QuerySet to introspect.\n\n Returns:\n List of field and annotation names in the order they will appear in\n ``qs.values()``.\n \"\"\"\n\n # We'll set the queryset to include all fields including calculated\n # aggregates using the same names which a values() queryset would return:\n v_qs = qs.values()\n\n field_names: list[str] = []\n field_names.extend(i.target.name for i in v_qs.query.select)\n field_names.extend(v_qs.query.extra_select.keys())\n field_names.extend(v_qs.query.annotation_select.keys())\n\n return field_names\n\n\ndef flatten_queryset(\n qs: QuerySet[Any],\n field_names: Iterable[str] | None = None,\n extra_verbose_names: Mapping[str, str] | None = None,\n) -> tuple[list[str], Iterable[Sequence[Any]]]:\n \"\"\"\n Convert a queryset into headers and row tuples for tabular export.\n\n By default the column headers are derived from the queryset's field\n names (as returned by ``get_field_names_from_queryset``) and the rows\n use ``values_list()`` for efficient iteration.\n\n If ``field_names`` is provided, only those fields are included and they\n are used to order both headers and row values.\n\n The ``extra_verbose_names`` mapping can override the verbose names for\n specific fields, including related lookups or calculated values.\n\n Args:\n qs: Base queryset to flatten.\n field_names: Optional explicit list of field names to include.\n extra_verbose_names: Optional mapping of field names to friendly\n column labels. This can be used to provide proper names for\n related lookups (for example,\n ``{\"institution__title\": \"Institution\"}``) or calculated values\n (for example, ``{\"items__count\": \"Item Count\"}``).\n\n Returns:\n A 2-tuple of ``(headers, rows)`` where ``headers`` is a list of\n column labels and ``rows`` is an iterable of sequences of values.\n \"\"\"\n\n if field_names is None:\n field_names = get_field_names_from_queryset(qs)\n\n # Headers will use the verbose names where available and fall back to the\n # field name if not (e.g. custom aggregate or extra fields):\n verbose_names = {i.name: i.verbose_name for i in qs.model._meta.fields}\n if extra_verbose_names is not None:\n verbose_names.update(extra_verbose_names)\n\n headers = [verbose_names.get(i, i) for i in field_names]\n\n return headers, qs.values_list(*field_names)\n\n\ndef convert_value_to_unicode(v: Any) -> str:\n \"\"\"\n Convert a value to a display-safe string for tabular export.\n\n ``None`` is rendered as an empty string. ``date`` and ``datetime``\n instances are converted using ``isoformat()``. All other values are\n coerced via ``force_str``.\n\n Args:\n v: Value to convert.\n\n Returns:\n String representation suitable for CSV, HTML, or XLSX output.\n \"\"\"\n\n if v is None:\n return \"\"\n elif hasattr(v, \"isoformat\"):\n return v.isoformat()\n else:\n return force_str(v)\n\n\ndef set_content_disposition(\n f: Callable[..., ResponseType],\n) -> Callable[..., ResponseType]:\n \"\"\"\n Decorator that applies a Content-Disposition header using the filename.\n\n The wrapped function must accept ``filename`` as its first positional\n argument and return an ``HttpResponse`` (or subclass). The decorator\n sets the ``Content-Disposition`` header using RFC 5987 encoding for the\n provided filename.\n\n Args:\n f: Callable that builds the HTTP response for a given filename.\n\n Returns:\n Wrapped callable that always sets ``Content-Disposition`` on the\n response.\n \"\"\"\n\n @wraps(f)\n def inner(filename: str, *args: Any, **kwargs: Any) -> ResponseType:\n response = f(filename, *args, **kwargs)\n # See RFC 5987 for the filename* spec:\n response[\"Content-Disposition\"] = \"attachment; filename*=UTF-8''%s\" % quote(\n filename\n )\n return response\n\n return inner\n\n\ndef return_debug_reponse(\n f: Callable[..., ResponseType],\n) -> Callable[..., ResponseType]:\n \"\"\"\n Decorator to swap export responses for an HTML debug table.\n\n When the ``TABULAR_RESPONSE_DEBUG`` setting is truthy, the wrapped\n function is not called. Instead ``export_to_debug_html_response`` is\n used and the ``Content-Disposition`` header is removed so the browser\n renders the table inline.\n\n Args:\n f: Export callable to wrap.\n\n Returns:\n Wrapped callable that either returns the original export response or\n an HTML debug response, depending on settings.\n \"\"\"\n\n @wraps(f)\n def inner(filename: str, *args: Any, **kwargs: Any) -> ResponseType:\n if not getattr(settings, \"TABULAR_RESPONSE_DEBUG\", False):\n return f(filename, *args, **kwargs)\n else:\n resp = export_to_debug_html_response(filename, *args, **kwargs)\n del resp[\"Content-Disposition\"] # Don't trigger a download\n return resp\n\n return inner\n\n\ndef export_to_debug_html_response(\n filename: str,\n headers: Iterable[Any],\n rows: Iterable[Sequence[Any]],\n) -> StreamingHttpResponse:\n \"\"\"\n Build an HTML table response for inspection of tabular export data.\n\n This is used when ``TABULAR_RESPONSE_DEBUG`` is enabled. It renders the\n headers and rows into a simple Bootstrap-styled HTML table and returns a\n ``StreamingHttpResponse``.\n\n Args:\n filename: Suggested filename for the export (kept for API parity).\n headers: Iterable of header labels.\n rows: Iterable of row sequences.\n\n Returns:\n StreamingHttpResponse streaming the HTML document.\n \"\"\"\n\n def output_generator():\n # Note the use of bytestrings to avoid unnecessary Unicode-bytes cycles:\n yield b\"\"\n yield b'TABULAR DEBUG '\n yield b'' # noqa\n yield b\"\"\n yield b\" \".join(\n convert_value_to_unicode(i).encode(\"utf-8\") for i in headers\n )\n yield b\" \"\n for row in rows:\n values = map(convert_value_to_unicode, row)\n values = [i.encode(\"utf-8\").replace(b\"\\n\", b\"%s \".join(values)\n yield b\" \"\n yield b\"
\n
Unacceptable Characters Report \n\n {% if errors %}\n
\n \n \n Asset \n Violations (line, column, char) \n \n \n \n {% for entry in errors %}\n \n \n \n {{ entry.asset }}\n \n \n \n \n {% for v in entry.violations %}\n \n Line {{ v.0 }}, Col {{ v.1 }}:\n {{ v.2|reprchar }}\n \n {% endfor %}\n \n \n \n {% endfor %}\n \n
\n {% else %}\n
No unacceptable characters were found.
\n {% endif %}\n
invalid\n submitted=timezone.now(),\n )\n\n assets = get_latest_transcription_data(\n Asset.objects.filter(pk__in=[self.asset.pk, bad_asset.pk])\n )\n\n with tempfile.TemporaryDirectory() as tmpdir:\n errors = do_bagit_export(assets, tmpdir, \"bad-bagit-no-request\")\n\n # helper should return a list, not an HttpResponse\n self.assertIsInstance(errors, list)\n self.assertEqual(len(errors), 1)\n self.assertEqual(errors[0][\"asset\"], bad_asset)\n\n # directory should have been removed\n mock_rmtree.assert_called_once_with(Path(tmpdir), ignore_errors=True)\n\n @override_settings(EXPORT_S3_BUCKET_NAME=None)\n @patch(\"exporter.views.shutil.rmtree\")\n @patch(\n \"pathlib.Path.exists\", return_value=False\n ) # force the exists() check to fail\n def test_do_bagit_export_validation_errors_no_export_dir(\n self,\n mock_exists,\n mock_rmtree,\n ):\n \"\"\"\n If validation fails and the export directory no longer exists,\n do_bagit_export should NOT attempt to call shutil.rmtree.\n \"\"\"\n bad_asset = create_asset(\n item=self.item,\n sequence=77,\n title=\"BadAssetNoDir\",\n download_url=DOWNLOAD_URL,\n resource_url=RESOURCE_URL,\n )\n Transcription.objects.create(\n asset=bad_asset,\n user=self.user,\n text=\"Oops\\u200b\", # zero-width space -> invalid\n submitted=timezone.now(),\n )\n\n assets = get_latest_transcription_data(\n Asset.objects.filter(pk__in=[self.asset.pk, bad_asset.pk])\n )\n\n with tempfile.TemporaryDirectory() as tmpdir:\n # invoke without request -> should return raw errors list\n errors = do_bagit_export(assets, tmpdir, \"bad-bagit-no-dir\")\n\n # exists() forced to False -> rmtree must not be called\n mock_rmtree.assert_not_called()\n self.assertIsInstance(errors, list)\n self.assertEqual(len(errors), 1)\n"
},
{
"path": "exporter/utils.py",
"content": "\"\"\"\nUtility helpers for validating outgoing text.\n\nThe primary public entry-point is `exporter.utils.validate_text_for_export`,\nwhich raises an `exporter.exceptions.UnacceptableCharacterError` when any\nnon-printable Unicode character is detected. Validation is performed per\nline so the caller receives the exact location of every problem character.\n\"\"\"\n\nfrom typing import List, Tuple\n\nfrom exporter.exceptions import UnacceptableCharacterError\n\n_WHITELIST = [\n \"\\t\", # Tab\n \"\\xa0\", # Non-breaking space\n \"\\u3000\", # Ideographic space; used in Chinese/Japanese/Korean\n \"\\u2003\", # em space (space the width of the 'm' character)\n]\n\n\ndef is_acceptable_character(character: str) -> bool:\n \"\"\"\n Return `True` when `character` is considered printable.\n\n The function simply wraps `str.isprintable()` so behaviour stays in sync\n with the official Unicode definition of *printable*.\n\n Args:\n character: A single Unicode character.\n\n Returns:\n `True` if the character is printable; otherwise `False`.\n \"\"\"\n\n return character.isprintable() or character in _WHITELIST\n\n\ndef find_unacceptable_characters(text: str) -> List[Tuple[int, int, str]]:\n \"\"\"\n Locate every non-printable character in *text*.\n\n The scan is performed line by line so that the exact position (line and\n column) of each offending character can be fed back to the caller.\n\n Args:\n text: The string to validate.\n\n Returns:\n A list of `(line_number, column_number, character)` triples. The list\n may contain duplicates because each instance of an invalid character is\n recorded individually.\n \"\"\"\n\n violations: List[Tuple[int, int, str]] = []\n\n for line_no, line in enumerate(text.splitlines(), start=1):\n for col_no, ch in enumerate(line, start=1):\n if not is_acceptable_character(ch):\n violations.append((line_no, col_no, ch))\n\n return violations\n\n\ndef validate_text_for_export(text: str) -> bool:\n \"\"\"\n Validate `text` and raise if it contains any unacceptable characters.\n\n Args:\n text: The text destined for export.\n\n Returns:\n `True` if the text is valid for export\n\n Raises:\n UnacceptableCharacterError: If at least one non-printable character is\n found.\n \"\"\"\n\n violations = find_unacceptable_characters(text)\n if violations:\n raise UnacceptableCharacterError(violations)\n return True\n\n\ndef remove_unacceptable_characters(text: str) -> str:\n \"\"\"\n Produce a copy of `text` with all non-printable characters removed.\n\n The removal uses the same acceptability rules as validation, ensuring the\n behaviour stays consistent with `is_acceptable_character()` and the shared\n whitelist. Standard line breaks are preserved. Characters considered\n unacceptable (i.e., not printable and not in the whitelist) are omitted\n from the result.\n\n Args:\n text: The input string to sanitize.\n\n Returns:\n A new string with all unacceptable characters removed.\n\n Notes:\n The scan mirrors `find_unacceptable_characters()` by operating\n line-by-line. Unlike validation, there is no error raised; the\n offending characters are dropped from the output. Newline characters\n (``\\\\n`` and ``\\\\r``) are preserved so the original line structure is\n maintained.\n \"\"\"\n\n cleaned_parts: List[str] = []\n for line in text.splitlines(keepends=True):\n # Keepends means any trailing '\\n'/'\\r\\n' is part of `line`.\n out_line_chars: List[str] = []\n for ch in line:\n # Preserve standard line breaks exactly as seen.\n if ch == \"\\n\" or ch == \"\\r\":\n out_line_chars.append(ch)\n continue\n if is_acceptable_character(ch):\n out_line_chars.append(ch)\n cleaned_parts.append(\"\".join(out_line_chars))\n return \"\".join(cleaned_parts)\n"
},
{
"path": "exporter/views.py",
"content": "import os\nimport re\nimport shutil\nimport tempfile\nfrom collections.abc import Iterable\nfrom logging import getLogger\nfrom pathlib import Path\nfrom typing import Any, List\n\nimport bagit\nimport boto3\nfrom django.conf import settings\nfrom django.contrib.admin.views.decorators import staff_member_required\nfrom django.contrib.postgres.aggregates.general import StringAgg\nfrom django.db.models import OuterRef, Subquery, TextField, Value\nfrom django.db.models.functions import Coalesce\nfrom django.db.models.query import QuerySet\nfrom django.http import (\n HttpRequest,\n HttpResponse,\n HttpResponseRedirect,\n)\nfrom django.shortcuts import render\nfrom django.utils.decorators import method_decorator\nfrom django.views.generic import TemplateView\n\nfrom concordia.models import (\n Asset,\n Campaign,\n Item,\n Project,\n Transcription,\n TranscriptionStatus,\n)\nfrom exporter.exceptions import UnacceptableCharacterError\nfrom exporter.tabular_export.core import export_to_csv_response, flatten_queryset\nfrom exporter.utils import validate_text_for_export\n\nlogger = getLogger(__name__)\n\n\ndef get_latest_transcription_data(\n asset_qs: QuerySet[Asset],\n) -> QuerySet[Asset]:\n \"\"\"\n Annotate each asset with its latest transcription text.\n\n The annotation is named ``latest_transcription`` and is derived from the\n most recent ``Transcription`` by primary key.\n\n Args:\n asset_qs:\n QuerySet[Asset] to annotate.\n\n Returns:\n QuerySet[Asset]: The input queryset annotated with\n ``latest_transcription``.\n \"\"\"\n latest_trans_subquery = (\n Transcription.objects.filter(asset=OuterRef(\"pk\"))\n .order_by(\"-pk\")\n .values(\"text\")\n )\n\n assets = asset_qs.annotate(\n latest_transcription=Coalesce(\n Subquery(latest_trans_subquery[:1]),\n Value(\"\", output_field=TextField()),\n )\n )\n return assets\n\n\ndef get_tag_values(asset_qs: QuerySet[Asset]) -> QuerySet[Asset]:\n \"\"\"\n Annotate each asset with a semicolon-joined string of tag values.\n\n The annotation is named ``tag_values`` and aggregates related tag text from\n ``userassettagcollection__tags__value``.\n\n Args:\n asset_qs:\n QuerySet[Asset] to annotate.\n\n Returns:\n QuerySet[Asset]: The input queryset annotated with ``tag_values``.\n \"\"\"\n assets = asset_qs.annotate(\n tag_values=StringAgg(\n \"userassettagcollection__tags__value\",\n \"; \",\n default=Value(\"\", output_field=TextField()),\n )\n )\n return assets\n\n\ndef remove_incomplete_items(item_qs: QuerySet[Item]) -> QuerySet[Asset]:\n \"\"\"\n Filter out items that are not fully completed and return their assets.\n\n An item is considered incomplete if any of its assets have a status of\n NOT_STARTED, IN_PROGRESS or SUBMITTED.\n\n Args:\n item_qs:\n QuerySet[Item] to check for completeness.\n\n Returns:\n QuerySet[Asset]: Assets belonging to completed items, ordered by\n project, item and sequence.\n \"\"\"\n incomplete_item_assets = Asset.objects.filter(\n item__in=item_qs,\n transcription_status__in=(\n TranscriptionStatus.NOT_STARTED,\n TranscriptionStatus.IN_PROGRESS,\n TranscriptionStatus.SUBMITTED,\n ),\n )\n item_qs = item_qs.exclude(asset__in=incomplete_item_assets)\n asset_qs = Asset.objects.filter(item__in=item_qs).order_by(\n \"item__project\", \"item\", \"sequence\"\n )\n return asset_qs\n\n\ndef get_original_asset_id(download_url: str) -> str:\n \"\"\"\n Derive a stable external asset identifier from a LOC download URL.\n\n For ``tile.loc.gov`` URLs a best-effort pattern is used to extract the\n identifier. Non-matching URLs are returned unchanged.\n\n Args:\n download_url:\n The asset's download URL.\n\n Returns:\n str: Identifier suitable for naming files inside the BagIt payload.\n\n Raises:\n ValueError: If the URL looks like ``tile.loc.gov`` but no ID is found.\n \"\"\"\n download_url = download_url.replace(\"https\", \"http\")\n if download_url.startswith(\"http://tile.loc.gov/\"):\n pattern = (\n r\"/service:([A-Za-z0-9:.\\-\\_]+)/\"\n + r\"|/master/([A-Za-z0-9/]+)([0-9.]+)\"\n + r\"|/public:[A-Za-z0-9]+:([A-Za-z0-9:.\\-_]+?)/\"\n )\n asset_id = re.search(pattern, download_url)\n if not asset_id:\n logger.error(\n \"Could not find a matching asset ID in download URL %s\",\n download_url,\n )\n raise ValueError(\n f\"Could not find a matching asset ID in download URL {download_url}\"\n )\n matching_asset_id = next((group for group in asset_id.groups() if group))\n logger.debug(\n \"Found asset ID %s in download URL %s\",\n matching_asset_id,\n download_url,\n )\n return matching_asset_id\n\n logger.warning(\n \"Download URL does not start with tile.loc.gov: %s\",\n download_url,\n )\n return download_url\n\n\ndef write_distinct_asset_resource_file(\n assets: Iterable[Any], export_base_dir: str | Path\n) -> None:\n \"\"\"\n Write a unique list of resource URLs for the provided assets.\n\n The file is named ``item-resource-urls.txt`` and written at the export\n root. Each line contains one distinct ``Asset.resource_url``.\n\n Args:\n assets:\n Iterable of asset identifiers or a QuerySet[Asset]. Passed to\n ``Asset.objects.filter(pk__in=assets)``.\n export_base_dir:\n Directory where the file should be created.\n\n Raises:\n AssertionError: If an asset has no ``resource_url``.\n \"\"\"\n asset_resource_file = os.path.join(export_base_dir, \"item-resource-urls.txt\")\n\n with open(asset_resource_file, \"a\") as f:\n distinct_resource_urls = (\n Asset.objects.filter(pk__in=assets)\n .order_by(\"resource_url\")\n .values_list(\"resource_url\", \"title\")\n .distinct(\"resource_url\")\n )\n\n for url, title in distinct_resource_urls:\n if url:\n f.write(url)\n f.write(\"\\n\")\n else:\n logger.error(\"No resource URL found for asset %s\", title)\n raise AssertionError\n\n\ndef do_bagit_export(\n assets: Iterable[Asset] | QuerySet[Asset],\n export_base_dir: str | Path,\n export_filename_base: str,\n request: HttpRequest | None = None,\n) -> HttpResponse | HttpResponseRedirect | List[dict[str, Any]]:\n \"\"\"\n Build and deliver a BagIt package for ``assets`` or report invalid chars.\n\n The function validates every asset's ``latest_transcription``. For each\n unacceptable character it records:\n - the asset ID\n - the 1-based line number\n - the 1-based column number\n - the offending character\n\n Behaviour:\n 1. Validation pass: files are written only after a transcription passes\n validation.\n 2. Failure(s): any files already written are removed. If ``request`` is\n supplied a template is rendered, otherwise the raw error list is\n returned.\n 3. All clear: a BagIt structure is created, zipped and either returned as a\n download or uploaded to S3.\n\n Args:\n assets:\n Iterable or QuerySet of ``Asset`` to export. Each must have\n ``download_url`` and ``latest_transcription``.\n export_base_dir:\n Temporary directory into which the BagIt hierarchy is built.\n export_filename_base:\n Base name (without ``.zip``) for the archive.\n request:\n Current request. If provided, an error template will be rendered\n when validation fails.\n\n Returns:\n HttpResponse | HttpResponseRedirect | list[dict[str, Any]]:\n\n - a download response when packaging locally\n - a redirect to the uploaded archive when S3 is configured\n - a list of validation errors when ``request`` is ``None`` and errors\n were found\n \"\"\"\n export_base_dir = Path(export_base_dir)\n errors: List[dict[str, Any]] = []\n\n # Validate every transcription before writing anything to disk\n for asset in assets:\n asset_id = get_original_asset_id(asset.download_url)\n asset_id = asset_id.replace(\":\", \"/\") # BagIt-safe path fragment\n\n transcription = asset.latest_transcription or \"\"\n try:\n validate_text_for_export(transcription)\n except UnacceptableCharacterError as err:\n errors.append({\"asset\": asset, \"violations\": err.violations})\n continue\n\n # Passed validation -> write the transcription file\n asset_path, asset_filename = os.path.split(asset_id)\n asset_dest_path = export_base_dir / asset_path\n asset_dest_path.mkdir(parents=True, exist_ok=True)\n\n if transcription:\n with open(asset_dest_path / f\"{asset_filename}.txt\", \"w\") as fp:\n fp.write(transcription)\n\n # If any errors -> cleanup and report\n if errors:\n if export_base_dir.exists():\n shutil.rmtree(export_base_dir, ignore_errors=True)\n\n if request is not None:\n return render(\n request,\n \"admin/exporter/unacceptable_character_report.html\",\n {\"errors\": errors},\n )\n return errors\n\n # All assets valid -> create BagIt, zip, upload / return\n write_distinct_asset_resource_file(assets, export_base_dir)\n\n bagit.make_bag(\n export_base_dir,\n {\n \"Content-Access\": \"web\",\n \"Content-Custodian\": \"dcms\",\n \"Content-Process\": \"crowdsourced\",\n \"Content-Type\": \"textual\",\n \"LC-Bag-Id\": export_filename_base,\n \"LC-Items\": f\"{len(assets)} transcriptions\",\n \"LC-Project\": \"gdccrowd\",\n \"License-Information\": \"Public domain\",\n },\n )\n\n archive_name = shutil.make_archive(\n str(export_base_dir), \"zip\", root_dir=export_base_dir\n )\n export_filename = f\"{export_filename_base}.zip\"\n\n s3_bucket = getattr(settings, \"EXPORT_S3_BUCKET_NAME\", None)\n if s3_bucket:\n logger.debug(\"Uploading exported bag to S3 bucket %s\", s3_bucket)\n s3 = boto3.resource(\"s3\")\n s3.Bucket(s3_bucket).upload_file(archive_name, export_filename)\n\n return HttpResponseRedirect(\n f\"https://{s3_bucket}.s3.amazonaws.com/{export_filename}\"\n )\n\n # Local download\n with open(archive_name, \"rb\") as zip_file:\n response = HttpResponse(zip_file, content_type=\"application/zip\")\n response[\"Content-Disposition\"] = f\"attachment; filename={export_filename}\"\n return response\n\n\nclass ExportCampaignToCSV(TemplateView):\n \"\"\"\n Stream a CSV of the most recent transcription for each asset in a campaign.\n\n Only the latest transcription text per asset is included. Tag values\n are aggregated into a semicolon-delimited string.\n \"\"\"\n\n @method_decorator(staff_member_required)\n def get(self, request: HttpRequest, *args, **kwargs) -> HttpResponse:\n \"\"\"\n Return a CSV response for the requested campaign.\n\n Args:\n request: Current HTTP request.\n\n Returns:\n HttpResponse: CSV content for the campaign.\n \"\"\"\n asset_qs: QuerySet[Asset] = Asset.objects.filter(\n item__project__campaign__slug=self.kwargs[\"campaign_slug\"]\n )\n assets: QuerySet[Asset] = get_latest_transcription_data(asset_qs)\n assets = get_tag_values(assets)\n\n headers, data = flatten_queryset(\n assets,\n field_names=[\n \"item__project__campaign__title\",\n \"item__project__title\",\n \"item__title\",\n \"item__item_id\",\n \"title\",\n \"id\",\n \"transcription_status\",\n \"download_url\",\n \"latest_transcription\",\n \"tag_values\",\n ],\n extra_verbose_names={\n \"item__project__campaign__title\": \"Campaign\",\n \"item__project__title\": \"Project\",\n \"item__title\": \"Item\",\n \"item__item_id\": \"ItemId\",\n \"item_id\": \"ItemId\",\n \"title\": \"Asset\",\n \"id\": \"AssetId\",\n \"transcription_status\": \"AssetStatus\",\n \"download_url\": \"DownloadUrl\",\n \"latest_transcription\": \"Transcription\",\n \"tag_values\": \"Tags\",\n },\n )\n\n logger.info(\"Exporting %s to csv\", self.kwargs[\"campaign_slug\"])\n return export_to_csv_response(\n \"%s.csv\" % self.kwargs[\"campaign_slug\"], headers, data\n )\n\n\nclass ExportItemToBagIt(TemplateView):\n \"\"\"\n Build a BagIt archive for a single item consisting of completed assets.\n\n Only assets with ``TranscriptionStatus.COMPLETED`` are included.\n \"\"\"\n\n @method_decorator(staff_member_required)\n def get(\n self, request: HttpRequest, *args, **kwargs\n ) -> HttpResponse | HttpResponseRedirect:\n \"\"\"\n Create and return a BagIt archive for the requested item.\n\n Args:\n request: Current HTTP request.\n\n Returns:\n HttpResponse | HttpResponseRedirect: Local zip download or redirect\n to S3.\n \"\"\"\n campaign_slug = self.kwargs[\"campaign_slug\"]\n project_slug = self.kwargs[\"project_slug\"]\n item_id = self.kwargs[\"item_id\"]\n\n asset_qs: QuerySet[Asset] = Asset.objects.filter(\n item__project__campaign__slug=campaign_slug,\n item__project__slug=project_slug,\n item__item_id=item_id,\n transcription_status=TranscriptionStatus.COMPLETED,\n ).order_by(\"sequence\")\n\n assets: QuerySet[Asset] = get_latest_transcription_data(asset_qs)\n\n campaign = Campaign.objects.get(slug__exact=campaign_slug)\n campaign_slug_dbv = campaign.slug\n project = Project.objects.get(campaign=campaign, slug__exact=project_slug)\n project_slug_dbv = project.slug\n item_id_dbv = Item.objects.get(item_id__exact=item_id).item_id\n\n export_filename_base = \"%s-%s-%s\" % (\n campaign_slug_dbv,\n project_slug_dbv,\n item_id_dbv,\n )\n\n with tempfile.TemporaryDirectory(\n prefix=export_filename_base\n ) as export_base_dir:\n return do_bagit_export(\n assets, export_base_dir, export_filename_base, request\n )\n\n\nclass ExportProjectToBagIt(TemplateView):\n \"\"\"\n Build a BagIt archive for a project consisting of completed items only.\n \"\"\"\n\n @method_decorator(staff_member_required)\n def get(\n self, request: HttpRequest, *args, **kwargs\n ) -> HttpResponse | HttpResponseRedirect:\n \"\"\"\n Create and return a BagIt archive for the requested project.\n\n Args:\n request: Current HTTP request.\n\n Returns:\n HttpResponse | HttpResponseRedirect: Local zip download or redirect\n to S3.\n \"\"\"\n campaign_slug = self.kwargs[\"campaign_slug\"]\n project_slug = self.kwargs[\"project_slug\"]\n\n item_qs: QuerySet[Item] = Item.objects.filter(\n project__campaign__slug=campaign_slug, project__slug=project_slug\n )\n asset_qs: QuerySet[Asset] = remove_incomplete_items(item_qs)\n assets: QuerySet[Asset] = get_latest_transcription_data(asset_qs)\n\n campaign = Campaign.objects.get(slug__exact=campaign_slug)\n campaign_slug_dbv = campaign.slug\n project = Project.objects.get(campaign=campaign, slug__exact=project_slug)\n project_slug_dbv = project.slug\n\n export_filename_base = \"%s-%s\" % (campaign_slug_dbv, project_slug_dbv)\n\n with tempfile.TemporaryDirectory(\n prefix=export_filename_base\n ) as export_base_dir:\n return do_bagit_export(\n assets, export_base_dir, export_filename_base, request\n )\n\n\nclass ExportCampaignToBagIt(TemplateView):\n \"\"\"\n Build a BagIt archive for a campaign consisting of completed items only.\n \"\"\"\n\n @method_decorator(staff_member_required)\n def get(\n self, request: HttpRequest, *args, **kwargs\n ) -> HttpResponse | HttpResponseRedirect:\n \"\"\"\n Create and return a BagIt archive for the requested campaign.\n\n Args:\n request: Current HTTP request.\n\n Returns:\n HttpResponse | HttpResponseRedirect: Local zip download or redirect\n to S3.\n \"\"\"\n campaign_slug = self.kwargs[\"campaign_slug\"]\n\n item_qs: QuerySet[Item] = Item.objects.filter(\n project__campaign__slug=campaign_slug\n )\n asset_qs: QuerySet[Asset] = remove_incomplete_items(item_qs)\n assets: QuerySet[Asset] = get_latest_transcription_data(asset_qs)\n\n campaign_slug_dbv = Campaign.objects.get(slug__exact=campaign_slug).slug\n\n export_filename_base = \"%s\" % (campaign_slug_dbv,)\n\n with tempfile.TemporaryDirectory(\n prefix=export_filename_base\n ) as export_base_dir:\n return do_bagit_export(\n assets, export_base_dir, export_filename_base, request\n )\n\n\nclass ExportProjectToCSV(TemplateView):\n \"\"\"\n Stream a CSV of the most recent transcription for each asset in a project.\n\n Only the latest transcription text per asset is included. Tag values\n are aggregated into a semicolon-delimited string.\n \"\"\"\n\n @method_decorator(staff_member_required)\n def get(self, request: HttpRequest, *args, **kwargs) -> HttpResponse:\n \"\"\"\n Return a CSV response for the requested project.\n\n Args:\n request: Current HTTP request.\n\n Returns:\n HttpResponse: CSV content for the project.\n \"\"\"\n campaign_slug = self.kwargs[\"campaign_slug\"]\n project_slug = self.kwargs[\"project_slug\"]\n\n campaign = Campaign.objects.get(slug__exact=campaign_slug)\n project = Project.objects.get(campaign=campaign, slug__exact=project_slug)\n\n asset_qs: QuerySet[Asset] = Asset.objects.filter(item__project=project)\n assets: QuerySet[Asset] = get_latest_transcription_data(asset_qs)\n assets = get_tag_values(assets)\n\n headers, data = flatten_queryset(\n assets,\n field_names=[\n \"item__project__campaign__title\",\n \"item__project__title\",\n \"item__title\",\n \"item__item_id\",\n \"title\",\n \"id\",\n \"transcription_status\",\n \"download_url\",\n \"latest_transcription\",\n \"tag_values\",\n ],\n extra_verbose_names={\n \"item__project__campaign__title\": \"Campaign\",\n \"item__project__title\": \"Project\",\n \"item__title\": \"Item\",\n \"item__item_id\": \"ItemId\",\n \"item_id\": \"ItemId\",\n \"title\": \"Asset\",\n \"id\": \"AssetId\",\n \"transcription_status\": \"AssetStatus\",\n \"download_url\": \"DownloadUrl\",\n \"latest_transcription\": \"Transcription\",\n \"tag_values\": \"Tags\",\n },\n )\n\n logger.info(\"Exporting %s to csv\", self.kwargs[\"project_slug\"])\n return export_to_csv_response(\n f\"{campaign.slug}-{project.slug}.csv\", headers, data\n )\n"
},
{
"path": "fixtures/original-static-pages.json",
"content": "[\n {\n \"fields\": {\n \"body\": \"crowd.loc.gov provides students of all ages with opportunities to explore unique historical documents from the collections of the Library of Congress.\\n\\nDeciphering and transcribing these documents can build students\\u2019 skills in close reading, examining historical context, and building interpretive consensus.\\n\\n## Teaching Ideas\\n\\n- As students transcribe a document, urge them to pay close attention to the language used in the document. What does that language tell them about who the audience for the document was? Was it a close friend or family member? A powerful person? A complete stranger? If the document is a letter, what clues can they find in the greeting and closing?\\n\\n- Prompt students to speculate about what the document\\u2019s author was trying to accomplish. What strategies, persuasive or otherwise, did the author use to accomplish their goal?\\n\\n- Challenge students to recreate the document in their own words, using today\\u2019s communication tools. What did your students say differently? What did they say that was the same?\\n\\n- Ask students to identify clues about the time and place in which the document was created. What do they know about what was going on at the time? How can they find out more? \\n\\n- Ask students to research and describe what happened as a result of this document being created. If they can\\u2019t identify what happened, ask them to speculate about what might have happened as a result of the document.\\n\\n- As students complete the transcription-reconciliation process, encourage them to take note of the ways they compared the differences between different transcriptions of the same document. What similarities and differences can they find between this process and the process of evaluating sources for a class project or research paper?\\n\\n## Primary Source Analysis\\n\\nThe documents in crowd.loc.gov are _primary sources_ \\u2014 the raw materials of history and culture. Analyzing primary sources like these can give students a powerful sense of the complexity of the past, and can guide them toward higher-order thinking and better critical thinking skills.\\n\\nUse the Library\\u2019s [primary source analysis tool and teacher\\u2019s guide to analyzing manuscripts](http://www.loc.gov/teachers/usingprimarysources/guides.html) to guide your students through an analysis of any of the documents in crowd.loc.gov.\\n\\n**A note about content**: The historical documents in the Library\\u2019s collections may include language or topics that aren\\u2019t appropriate for your students, or that your students might find especially difficult to engage with. You may want to review documents before assigning them, or to use some of the strategies explored here.\\n\\n## Campaign Specific Resources\\n\\n### Letters to Lincoln (Featured Challenge for 2018!)\\n\\n- [\\\"I Do Solemnly Swear...\\\" Presidential Inaugurations](https://www.loc.gov/rr/program/bib/inaugurations/index.html)\\n- [The Library of Congress Celebrates the Songs of America](https://www.loc.gov/collections/songs-of-america/about-this-collection/)\\n- [Additional Primary Materials](http://www.loc.gov/teachers/classroommaterials/primarysourcesets/lincoln/)\\n\\n### Branch Rickey: Changing the Game\\n\\n- Lesson Plan: [Baseball, Race and Ethnicity: Rounding the Bases](http://www.loc.gov/teachers/classroommaterials/lessons/bases/)\\n- Lesson Plan: [Baseball, Race Relations and Jackie Robinson](http://www.loc.gov/teachers/classroommaterials/lessons/robinson/)\\n- Primary Source Set: [Baseball Across a Divided Society](http://www.loc.gov/teachers/classroommaterials/primarysourcesets/baseball/)\\n\\n### Civil War Soldiers: \\\"Disabled but not disheartened\\\"\\n\\n- Lesson plan: [The Civil War Through a Child's Eye](http://www.loc.gov/teachers/classroommaterials/lessons/childs-eye/)\\n- Teacher's guide: [The Civil War: The Nation Moves Towards War, 1850-61](http://www.loc.gov/teachers/classroommaterials/primarysourcesets/civil-war-approach/)\\n\\n### Clara Barton: \\\"Angel of the Battlefield\\\"\\n\\n- [Clara Barton Missing Soldiers Office Museum External additional resources](http://www.clarabartonmuseum.org/learn/studentresearch/)\\n\\n### Mary Church Terrell: Advocate for African Americans and Women\\n\\n- Lesson plan: [Suffrage Strategies: Voices for Votes](http://www.loc.gov/teachers/classroommaterials/lessons/suffrage/)\\n- [http://www.loc.gov/teachers/classroommaterials/lessons/strivings/](http://www.loc.gov/teachers/classroommaterials/lessons/strivings/)\\n- [Votes for Women: Selections from the National American Woman Suffrage Association Collection, 1848 to 1921](https://www.loc.gov/teachers/classroommaterials/connections/votes-women/)\\n- [Civil Rights, information for Students](http://www.loc.gov/teachers/classroommaterials/themes/civil-rights/students.html)\\n- [From Slavery to Civil Rights: A Timeline of African American History](https://www.loc.gov/teachers/classroommaterials/presentationsandactivities/presentations/civil-rights/) \\n- [Segregation: From Jim Crow to Linda Brown](http://www.loc.gov/teachers/classroommaterials/lessons/jimcrow/)\",\n \"created_on\": \"2018-11-26T22:00:43.688Z\",\n \"path\": \"/for-educators/\",\n \"title\": \"Resources for Educators\",\n \"updated_on\": \"2018-11-26T22:00:43.696Z\"\n },\n \"model\": \"concordia.simplepage\",\n \"pk\": 1\n },\n {\n \"fields\": {\n \"body\": \"## Join us at the Library for the 155th Anniversary of the Gettysburg Address\\n\\n---\\n\\nOn November 19, 1863, Abraham Lincoln delivered the Gettysburg Address in Gettysburg, Pennsylvania. The Library of Congress will mark the 155th anniversary of this historic speech with a one-day celebration, featuring an exhibition of the earliest known draft of the speech, and a Letters to Lincoln Challenge transcribe-a-thon for volunteers on and off site!\\n\\n### Schedule\\n\\n- 08:30am Jefferson Building doors open!\\n- 10:00am Welcome remarks by Dr. Carla Hayden, 14th Librarian of Congress\\n- 10:05am Special talk about the background of the Gettysburg Address by historian and curator Michelle Krowl\\n- 10:15am Gettysburg Address delivered by student orator Christian Melgar\\n- 10:25am Details of the Letters to Lincoln Challenge for onsite and remote participants\\n- 10:30am Nicolay copy of the Gettysburg Address unveiled \\n- 10:35am Letters to Lincoln Challenge transcribe-a-thon begins \\n- 10:35am-1:30pm Letters to Lincoln Challenge Transcribe-a-thon\\n- 4:30pm Jefferson Building doors close for the day \\n\\n#### In the area on November 19th?\\n\\nJoin us in the Great Hall of the Jefferson Building (Library of Congress) to have the rare opportunity to see the Nicolay copy of the Gettysburg Address in person. Dr. Hayden will kick things off at 10am, followed by a reading of the Address by a student orator, and a special talk about Lincoln and the Gettysburg Address, by historian and curator Michelle Krowl. After the Gettysburg Address is revealed at 10:30 various hands on transcription and learning opportunities will be available. \\n\\n#### Off site but online?\\n\\nWe\\u2019ll livestream the Librarian and curator\\u2019s talks, and the reading of the Gettysburg address from 10:00-10:30 [via this link](https://www.youtube.com/loc). After that people at the Library and online can participate in the #LettersToLincoln challenge right here on [crowd.loc.gov](/campaigns/letters-to-lincoln/)! Choose an item from the Letters to Lincoln Campaign to transcribe, review, and/or tag. So far we've made material from the 1830s through 1850s available, but for our transcribe-a-thon we'll release materials from 1860, 1861, and 1862, spanning part of the Civil War and important moments in Lincoln's career and the history of the nation. Join in the discussion on [History Hub](https://historyhub.history.gov/community/crowd-loc) and [Twitter](https://twitter.com/Crowd_LOC)! \\n\\nYou can [check in live](https://historyhub.history.gov/community/crowd-loc/blog/2018/11/14/transcribe-with-us-nov-19-the-155th-anniversary-of-the-gettysburg-address) with a Community Manager on History Hub throughout the transcribe-a-thon from 10:30am to 1:30pm EST. \\n\\n#### Opportunities for students near and far\\n\\nTune in at 10:00am for the livestream or join us in person to see the Gettysburg address, hear talks, and do some hands on activities and online transcriptions. Students onsite or in their own classrooms will be invited to transcribe, tag, and review documents received by Abraham Lincoln throughout his career. You can transcribe as a group or challenge your students to transcribe on their own or in pairs. All transcriptions are reviewed by at least one other volunteer, so don't be shy to try! If you or your students are finding it hard to read something, try finding something to review first. This is a great way to \\\"get your eye in\\\" and learn from others how to read these original documents.\\n\\nTo confirm participation for your class or students, please e-mail the Community Managers at [crowd@loc.gov](mailto:crowd@loc.gov) for further instructions. [Visit History Hub](https://historyhub.history.gov/community/crowd-loc/blog/2018/11/14/transcribe-with-us-nov-19-the-155th-anniversary-of-the-gettysburg-address) for your transcribe-a-thon pack including details on how to take part. \\n\\n## Letters to Lincoln Challenge\\n\\n---\\n\\n\\n\\n### Help us transcribe 10,000+ items from the Abraham Lincoln Papers by the end of 2018!\\n\\n#### A grand challenge: why we're asking you to join us\\n\\nAround half of the digitized Abraham Lincoln Papers, primarily materials written by Lincoln, have been transcribed by other volunteers at Knox College and elsewhere, and are already keyword searchable at loc.gov. However, there remain 10,000+ items including letters and other materials sent to him that are not yet keyword searchable. Completing the [Letters to Lincoln Challenge](https://crowd.loc.gov/campaigns/letters-to-lincoln/) will make all of the digital Lincoln Papers word-searchable and accessible to future readers. Just imagine the possibilities--from new research to local connections--that will be possible once we've achieved this goal. Thank you in advance for sharing your time with us. Your Community Managers, reference librarians and curatorial staff here at the Library of Congress will be cheering you on with bonus historical context and resources all along the way, as well as some special rewards for goals met!\\n\\n#### So, whatβs the challenge? Our first milestone was completion of all the material in the first three Campaign projects: \\\"1830-1839, first forays in politics and law,\\\" \\\"1840-1849, marriage, election to Congress,\\\" and \\\"1850-1857: death and birth of children, and re-entry to politics\\\" by November 1st. Our next *updated* challenge to you is to transcribe and review all 646 pages in the \\\"1858-1859 Presidential Nomination\\\" project by midnight on November 6th, election day!\\n\\nCan you transcribe even just one letter and share the challenge with one friend to help push toward our goal? When the project completes weβll move onto the next exciting decade of Lincoln's life, the 1850s when he returned to politics.\\n\\n#### What are the Letters to Lincoln?\\n\\nYou might guess that the Abraham Lincoln Papers include materials written in his own hand, but did you know the collection contains correspondence sent to Abraham Lincoln throughout his life and political career? Here's a taste of what you'll find: a range of materials by writers ranging from friends and associates from Lincolnβs Springfield days, well-known political figures and reformers, constituents writing to their President, and even the occasional document in Lincoln's own hand. Read the concerns and requests of nineteenth-century Americans and international correspondents.\\n\\n## Library of Congress News\\n\\n---\\n\\nEnabling Discovery of unique treasures at the Library of Congress [Press release](https://www.loc.gov/item/prn-18-134/crowdsourcing-tool-enables-discovery-of-unique-treasures-at-the-library-of-congress/2018-10-24/)\\n\\nAnnouncing crowd.loc.gov: [Here we go!](https://blogs.loc.gov/thesignal/2018/10/lets-go-explore-transcribe-and-tag-at-crowd-loc-gov/)\\n\\n[Connecting crowdsourcing](https://blogs.loc.gov/thesignal/2018/10/new-strategy-new-crowd-new-team/) to the new Library of Congress 2019-2023 Strategic Plan and Digital Strategy\\n\\n## Press coverage\\n\\n---\\n\\nMental Floss - 28 October 2018 - [The Library of Congress Needs Help Transcribing Lincoln's Letters and Other Historic Documents](http://mentalfloss.com/article/561842/library-congress-needs-help-transcribing-lincolns-letters-and-other-historic)\",\n \"created_on\": \"2018-11-26T22:00:43.700Z\",\n \"path\": \"/latest/\",\n \"title\": \"Latest News\",\n \"updated_on\": \"2018-11-26T22:00:43.702Z\"\n },\n \"model\": \"concordia.simplepage\",\n \"pk\": 2\n },\n {\n \"fields\": {\n \"body\": \"\\n
\\n
\\n
\\n
\\n
Welcome to crowd.loc.gov \\n
Here you will find a guide to help you get started. \\n
Learn more » \\n
\\n
\\n
\\n
\\n
\\n \\n\\n
\\n
\\n
\\n crowd.loc.gov is an online transcription platform where anyone with an internet connection can transcribe documents from the Library of Congress\\u2019 digitized collections. We welcome anyone interested in making non-machine readable resources fully word searchable to contribute.\\n
\\n
\\n Crowdsourcing invites members of the public, non-specialists and specialists alike, to help make data more usable and discoverable. Crowdsourcing at the Library of Congress invites unpaid volunteers to explore collections while gaining new skills, for example, learning to read older forms of handwriting such as cursive.\\n
\\n
\\n
\\n\\n
\\n
\\n \\n
\\n
\\n Anyone who wants to help the Library make its collections more discoverable online. Anyone who is interested in history, cultural heritage, literature, languages, art, sciences, and much more. Anyone who wants to be a virtual volunteer, exploring collections and transcribing at their own pace and at times that are convenient for them. Students of all ages who want to help the Library and learn new skills.\\n\\n
\\n
\\n\\n
\\n
\\n \\n
\\n
\\n You'll collaborate with other volunteers to transcribe and review collections. We ask for at least one person to transcribe and a different person to review each transcription. If you find that a transcription needs a few corrections while you're reviewing, you can edit that page. Another person will then review your new edits. Sometimes more than one person will contribute to transcribe an image; such as if you find an image with a transcription that needs more work. We think of this negotiated editing process as a way to get the best version of a transcription and help solve different challenges for each image.\\n\\n
\\n
\\n\\n
\\n\\n
\\n \\n
\\n
\\n Materials in crowd.loc.gov represent the diversity of the Library\\u2019s collections and are selected from across the Library of Congress\\u2019s curatorial divisions. You\\u2019ll encounter presidential papers, materials from the women's suffrage, abolition and other movements, the work of American poets, such as Walt Whitman, and much more. We will add new content regularly:
sign up for our newsletter to hear about new Campaigns and Challenges.\\n\\n
\\n
\\n\\n
\\n\\n
\\n \\n
\\n
\\n Follow the transcribe, review, or tag links at the top of the page. Our goal is to make transcriptions that are readable to computers and humans, with minimal markup, not attempting to recreate the layout of the original images. Quick tips are available within the transcription interface.\\n\\n
\\n
\\n\\n
\\n
\\n \\n
\\n
\\n \\nRegistering for an account is optional, but gives you access to the tagging and reviewing features of crowd.loc.gov. Go to Register at the top of the homepage to make an account. Create a username, which will be visible to other volunteers and users of the site. Enter your email address in the \\u201cEmail\\u201d field (this will not be visible to or shared with other users), and in the password field create a unique password with a combination of capital and lowercase letters, numbers and special characters such as #, $, !, or %.\\n\\nYou can also register for a separate account on the
History Hub discussion forum , where you can discuss the material you are transcribing or your experience of encountering primary sources. Feel free to raise questions or concerns, especially if you think other volunteers might be able to respond or benefit from your post. History Hub is a moderated forum. The Community Managers will check in regularly to approve comments and engage in discussion, and will try to answer questions about the project or the collections within 3-5 business days. Reference Librarians and curators will also answer some questions.\\n\\n
\\n
\\n\\n
\\n
\\n \\n
\\n
\\n \\nIf you've forgotten your password click Login, then \\\"Forgot my password\\\". You will receive an email with a link asking you to reset your password. You may also change your password within your profile.\\n
\\n
\\n
\\n
\\n \\n
\\n
\\n \\nYour email address gives us the ability to support you. Community managers are here to help with account administration like changes to your profile, troubleshooting any issues with contributing to a transcription, or to answer general questions. We will never share your information with other institutions or individuals. At registration you can opt in to receive email updates on crowd.loc.gov campaigns and features \\u2013 you can also
register for emails from us here .\\n
\\n
\\n
\\n
\\n \\n
\\n
\\n \\nThe data contributed by volunteers like you can be used in many different ways. We are giving back to our community by making this data public. All contributions to this application are released into the public domain. Anyone is free to use this data set in any way they want. The data produced by volunteers is also free to reuse. If you need help accessing the data or want to share news of your research with the crowd.loc.gov community, please contact the Community Managers at
crowd@loc.gov . The transcriptions produced on crowd.loc.gov will typically be published in the Library catalog on loc.gov within a year of a Campaign's completion. \\n
\\n
\\n
\\n
\\n \\n
\\n
\\n Tags are an experimental feature. Tags can be used to identify people, places or things in documents that are not already identified in the page or asset\\u2019s metadata on
loc.gov . We want to understand how volunteers like to use tags. We also want to understand whether tags can someday be included in the metadata on the Library catalog to make items discoverable through search terms that are not represented in the existing metadata or the transcriptions we will produce on crowd.loc.gov.\\n\\n
\\n
\\n\\n
\\n
\\n \\n
\\n
\\nThe language and terminology used in the historical materials on this site reflect the context and culture of their creators, and may include words, phrases, and attitudes that would now be deemed insensitive, inappropriate or factually inaccurate, or may not be appropriate for all ages. Views expressed in historical documents do not reflect the views of the Library of Congress. Because the purpose of crowd.loc.gov is to make the Library\\u2019s collections searchable, we ask that all original content be transcribed as it appears in the original material. If you find some material offensive or upsetting, please choose something else to transcribe. If you have questions or comments regarding the material you encounter during your participation here, please contact a Community Manager via
crowd.loc.gov@loc.gov , the
Contact Us or join and start a new conversation on the
History Hub discussion forum .\\n\\n
\\n
\\n\\n
\\n\\n
\\n \\n
\\n
\\nBecause crowd.loc.gov invites you to transcribe documents, it is best experienced on a device with a large or full sized keyboard. A desktop computer or laptop is best; a tablet with keyboard should work. Unfortunately, phones are not yet supported. We recommend an external mouse for most precise zoom. We support the two most recent versions of major browsers. You\\u2019ll have the best experience if you use Chrome, Firefox, Edge, and Safari browsers. The site will not work as designed on the Internet Explorer browser.\\n\\n
\\n
\\n\\n
\\n
\\n \\n
\\n
\\nCrowd.loc.gov runs on Concordia, new open source software developed by the Library of Congress to power crowdsourced transcription projects. The code is visible and free to reuse:
Visit our Github repository for more information. The platform was built utilizing user-centered design principles based around building trust and approachability. This project is a partnership between the Library and a growing community of volunteers who help us to iteratively improve the platform. Everyone is welcome to take part in transcription and tagging and to give feedback about how we can improve the code base and the project itself. Be in touch!\\n\\n
\\n
\\n\\n
\\n
\\n \\n
\\n
\\n
\\nA detailed explanation of the Library\\u2019s Privacy Policy including what kinds of data we collect and store, and what we use to track your session while you are on a Library website is available at this link, and in the footer of this page under the \\u201cLegal\\u201d link button.\\n
\\n
\\nYou do not have to register an account on crowd.loc.gov in order to transcribe, but you can register if you would like to review or tag. In order to make sure a transcription is submitted by a real human, anonymous users will be prompted to fill in a captcha before their first submission will be accepted. The Library\\u2019s captcha is an image of a few letters and numbers that you need to transcribe into the box below the image.\\n
\\n
\\nA session cookie will be used in your browser while you are transcribing so that you do not need to enter a captcha every time you work on a page. Session cookies for anonymous users are limited to 24 hours, so you will only be prompted to enter a captcha once a session.\\n
\\n
\\nSession cookies are used for registered users too, so that your contributions can be saved to your account. Check out your user profile to see how many pages you have transcribed, tagged and reviewed. Registered user session cookies last two weeks.\\n
\\n
\\n
\\n\\n
\\n
\\n \\n
\\n
\\nThe Library of Congress has long invested in building digitized collections and making them searchable. The Library\\u2019s first attempt recruiting members of the public to increase findability on our website began in 2008 when the Photography and Prints Division published thousands of photographs on
Flickr Commons . This long-running project invites visitors to help identify people and places in the photographs and, once verified, this rich information is used to enhance the online catalog and improve access for all users. Two additional crowdsourcing efforts within the Library include
Roll the Credits and
Beyond Words , projects that invited people to transcribe credit captions from television programs, and identify cartoons and photographs in the Library's historic newspaper collections respectively.\\n\\n
\\n
\\n\\n
\\n
\\n \\n
\\n
\\nIn addition to the Library of Congress\\u2019 own history of varied participatory projects, other cultural heritage institutions with established transcription programs have paved the way for crowd.loc.gov. Projects at the National Archives and Records Administration (NARA), the Smithsonian, the New York Public Library, Zooniverse.org, From the Page, and others have developed workflows and user engagement strategies that this platform leverages and builds upon. Concordia deploys a different architecture to these existing models of crowdsourced transcription, and aims to provide simple data structures and easier project implementation for cultural heritage institutions and other people who want to set up their own crowdsourced transcription projects.\\n
\\n
\\n
\\n
\\n
\\n
\\n
\\n
\\n
Find guides to help you tag, transcribe, and review. \\n
\\n
\\n
\\n
\\n
\\n
\\n
\\n
Join the conversation on History Hub. We'll take you there in a new tab. \\n
\\n
\\n
\\n
\\n
\\n
\\n
\\n
Have a problem with a page or an image? Want to see a new feature? Send us a message. \\n
\\n
\\n
\\n
\\n
\\n
\\n\\n
\\nIn addition to transcribing, you can review and edit transcriptions created by your fellow volunteers. A reviewer\\u2019s task is to read the entire transcription and carefully compare it against the image of the document. If you want to become a reviewer, please register an account.\\n
\\n\\n
A good transcription: \\n\\n
\\nWhen a transcription is accurate and you do not need to make any changes, click the \\\"Approve\\\" button to mark the page as complete. The page will no longer be available for transcription, but you and other volunteers can still read the document and add tags.\\n
\\n\\n
\\nA transcription that needs more work:\\n \\n
\\nWhile reviewing, you can change a transcription to fix errors or add missing material. Remember, do not edit the spelling and grammar of the original document, but do correct any spelling errors or misreadings created by the transcriber. When youβre done, click the \\\"Save\\\" and \\\"Submit\\\" buttons. Another volunteer will then need to review the page. A page is complete when a reviewer clicks \\\"Accept\\\" without making changes.\\n
\\n
\\n
\\n
\\n
\\n
\\nYou can use tags however you like, so long as you avoid using offensive, degrading or hurtful language about other individuals or groups of people. Read the Library\\u2019s commenting policy here . Here are some ways you might consider tagging:\\n
\\n
\\nIf you transcribe an important word in a document, such as somebody\\u2019s name, and the original author spelled the name incorrectly, you can add a tag to provide the correct name using the \\u201cTag\\u201d button. \\n\\nSometimes writers use nicknames or code words. If you know or can correctly identify the full name or subject using contextual information from the larger document or collection, please tag this information using the \\u201cTag\\u201d button.\\n \\n Are you interested in documents mentioning cats? Use the \\u201cTag\\u201d button to tag all pages that mention cats. Other examples include \\u201cCivil War\\u201d, \\u201cCooking\\u201d, \\u201cSports\\u201d. You can apply whatever tags you like. \\n Keep tags as short as you can and use whole words instead of abbreviations. This will make it easier for other people to understand your tags and to reuse them elsewhere in the collection. \\n Tagging is an experimental feature. Tags might one day go back into the Library website, but for now they will only be used as a method to explore items within crowd.loc.gov.\\n \\n \\n
\\n
\\n
\\n
\\n
\\nWelcome to crowd.loc.gov! Our goal is to make documents word-searchable in the Library of Congress catalog. Help us transcribe original items, many of which have never been transcribed before. Anyone can transcribe, you don\\u2019t need an account, but registered volunteers can access the review and tag features. You can choose which tasks you want to do, and what pages you want to work on. To create an account, go to our registration page . Let's get started!\\n
\\n
\\nChoose what to transcribe. You can either click the \\\"Let's Go\\\" button on the homepage or explore our various campaigns .\\n \\n\\nTranscribe what you can into the white box on the right. Transcribe lines in the order they appear and preserve line breaks. This will make the transcriptions useful for researchers. If you see multiple pages, such as an open notebook with two pages, just transcribe the content in the order it appears.\\n \\n\\nDon\\u2019t reproduce features such as underlining, bold text, line breaks or indentation in your transcription. Only words will be searchable in these transcriptions--not formatting. See the How to transcribe section for more information.\\n \\n\\nClick \\u201cSave\\u201d as you go to save your work in progress. Always click \\u201cSave\\u201d before you move from transcribing to tagging or moving to another page.\\n \\n\\nClick \\\"Save\\\" and \\u201cSubmit for review\\u201d when you are done transcribing and are ready to move on to the next page or to reviewing or tagging. Anonymous users will be prompted to submit a captcha.\\n \\n\\nYou can explore collections by clicking the Campaigns link at the top of the screen.\\n \\n\\nRegistered volunteers can review other people\\u2019s transcriptions. To start reviewing pages, select an item and click an image titled \\u201cReview\\u201d or filter the images by \\\"Review\\\". Read a page carefully and decide if the transcription looks correct. When a transcription is accurate and you do not need to make any changes, click the \\\"Approve\\\" button. The page will now be marked as complete. If you need to make changes start editing the page by clicking \\\"Edit\\\". Be sure to click \\u201cSave\\u201d periodically, and then click \\u201cFinish\\u201d. Changes you submit will be reviewed by another volunteer. You cannot review your own work. This process continues until someone Completes a transcription by clicking \\u201cApprove\\u201d without making changes.\\n \\n\\nIf you have a question or comment about how crowd.loc.gov works, or about a Campaign, item or page, connect with our team and other volunteers on History Hub , a public online forum where everyone can join in the discussion. If you would prefer to email a Community Manager send us a message on our Contact Us page. \\n \\n \\n
\\n
\\n
\\n
\\nWe transcribe to improve search functionality. Our goal is to make documents word-searchable in the Library of Congress catalog, which means typing transcriptions that can be read by that computer system, as well as by humans. Most handwriting and some typed text cannot be automatically and accurately translated into machine-readable text using current technologies -- that\\u2019s why we need your help!\\n
\\n
\\nWe ask you to transcribe a document roughly as it appears on the page. Preserve line breaks, except in cases where words are broken over two lines. Our main goal is to capture all of the words on these pages, so broken words are not helpful for search. Using line breaks to roughly capture the layout of the page helps reviewers (other volunteers) check transcriptions. These will be viewable beside the original images in the catalog, so anyone who is interested in the physical layout of the original document will be able to see it.\\n
\\n\\n
All in order: \\n
\\nTranscribe text in the order it appears.\\n
\\n
\\nSpelling:\\n \\n
\\nPreserve original spelling unless the author seems to have made a minor error, such as writing \\u201cteh\\u201d instead of \\u201cthe\\u201d. If a misspelling will impact the searchability of the document, use a tag to add the correct spelling. Example:\\n
\\n An author wrote \\u201cWilla Kather\\u201d instead of \\u201cWilla Cather\\u201d. Transcribe Willa Kather, and tag \\u201cWilla Cather\\u201d. \\n \\n\\n\\n
Insertions: \\n
\\nWhen text has been inserted or added later, but should be read as part of a sentence, bring it down into the line and type it in the order you would read it aloud.\\n
\\n
\\nLine-breaks: \\n
\\nDo not reproduce words broken between two lines. Write library rather than li-brary, kitten rather than kit-ten. Otherwise hit \\\"enter\\\" or \\\"return\\\" at the end of a line, to roughly mirror the layout of the original document. This will help reviewers in crowd.loc.gov to easily compare your transcription with the image.\\n
\\n
\\nEmphasis:\\n \\n
\\nThe Library catalog cannot search for bold, italic, underlined or superscript text, so even when you see these features please transcribe the words without any styling.\\n
\\n
\\nIllegible:\\n \\n
\\nIllegible text is anything you can\\u2019t read because a page is damaged, text is heavily crossed out or you can\\u2019t tell what the author has written. If there is a word or a string of words you cannot read use a pair of square brackets around an empty space [ ]. Example:\\n
\\n
\\n \\\"I have [ ] loved coffee ice cream\\\" \\n \\n
\\nIf you can read any letters or parts of words transcribe what you can and use question marks for the remaining letters or words. Examples:\\n
\\n
\\n \\\"I have [a?????] loved coffee ice cream\\\" \\n \\\"I have [a?] loved coffee ice cream\\\" \\n \\n
\\nIf you cannot read a word or phrase that\\u2019s ok. Another volunteer may be able to identify the missing letters and update your transcription. If there is a lot of text you cannot read consider looking for another page that you can decipher better.\\n
\\n
Deletions: \\n
\\nIf you can read crossed out or otherwise deleted text, transcribe the deleted words within a pair of square brackets. Example:\\n
\\n
\\u201cI have always loved [vanilla] coffee ice cream.\\u201d \\n\\n
Marginalia: \\n\\n
\\nMarginalia is text written in the space around the main block of text. It is usually a comment on the main body text but can also be unrelated. It is different from an insertion, because it cannot be directly inserted into the main text and still make sense when read aloud. Use a pair of square brackets and asterisk [*] around marginalia text and order it within the transcription where it makes the most sense (or at the end of the transcription if it appears unrelated). Example:\\n
\\n
\\n I have always loved coffee ice cream. Last summer I made my own using a recipe from the 1970s. It was the creamiest coffee ice cream I ever ate. No one else in my family likes that flavor. Oh well, more for me! [*Brazil was the largest coffee producing country in the world in 2017*] \\n\\n\\n
Printed and typed text: \\n
\\nSome material in crowd.loc.gov was created on a typewriter or printed. If we have included it here it is because the text is not machine-readable. A computer using Optical Character Recognition (OCR) technology cannot create an accurate word-searchable transcription. Examples include the scouting reports of Branch Rickey, which are typed on thin paper and are often too fuzzy for successful OCR. Similarly, mixed materials containing manuscript and print have not been run through OCR, so please transcribe letterhead and any other printed features that will shed light on where a document was created.\\n
\\n\\n
When not to transcribe printed text: \\n
\\nSome calendars and diaries contain many pages of pre-printed almanacs or other text that is probably machine-readable and therefore should not be transcribed as part of this project. It might be interesting to copy the first page from a repeating template in a diary or journal, but this is not the core text we are aiming to capture. However, if you want to transcribe it, feel free. Alternatively, click \\\"Nothing to transcribe\\\". This button should also be used for archival folders, blank pages, and pictorial images.\\n
\\n \\n
Tables, graphics, images: \\n
\\nSome documents will contain tables. Transcribe these in a way that will make them relatively easy for a reviewer to check over, but don't try to capture the exact layout of the data. The material will go back into the catalog without styling. Don't include notes or descriptions of visual features. If you would like to describe images, watermarks, stamps, or any other non-text features, feel free to use the tagging function. Register for an account if you want to add tags!\\n
\\n
\\n
Vite + React \n \n \n
\n \n \n\n"
},
{
"path": "frontend/package.json",
"content": "{\n \"name\": \"frontend\",\n \"private\": true,\n \"version\": \"0.0.0\",\n \"type\": \"module\",\n \"scripts\": {\n \"dev\": \"vite\",\n \"build\": \"vite build\",\n \"lint\": \"eslint .\",\n \"preview\": \"vite preview\"\n },\n \"dependencies\": {\n \"@reduxjs/toolkit\": \"^2.8.2\",\n \"lodash.debounce\": \"^4.0.8\",\n \"openseadragon\": \"^5.0.1\",\n \"openseadragon-filters\": \"^2.2.0\",\n \"react\": \"^19.1.0\",\n \"react-bootstrap\": \"^2.10.10\",\n \"react-dom\": \"^19.1.0\",\n \"react-redux\": \"^9.2.0\",\n \"react-router-dom\": \"^6.30.3\",\n \"screenfull\": \"^6.0.2\",\n \"split.js\": \"^1.6.5\"\n },\n \"devDependencies\": {\n \"@eslint/js\": \"^9.25.0\",\n \"@types/react\": \"^19.1.2\",\n \"@types/react-dom\": \"^19.1.2\",\n \"@vitejs/plugin-react\": \"^4.4.1\",\n \"eslint\": \"^9.25.0\",\n \"eslint-plugin-react-hooks\": \"^5.2.0\",\n \"eslint-plugin-react-refresh\": \"^0.4.19\",\n \"globals\": \"^16.0.0\",\n \"vite\": \"^7.3.2\",\n \"vite-plugin-static-copy\": \"^3.1.2\"\n }\n}\n"
},
{
"path": "frontend/src/App.jsx",
"content": "import React, {useEffect, useState} from 'react';\nimport {HashRouter, Routes, Route, Link, useParams} from 'react-router-dom';\nimport ViewerSplit from './ViewerSplit';\n\n/**\n * Fetches a JSON endpoint and displays the response.\n *\n * Useful as a temporary inspector while APIs and UI are evolving.\n *\n * @param {Object} props\n * @param {string} props.endpoint\n * Relative or absolute API URL to request.\n * @param {string} [props.method=\"GET\"]\n * HTTP method to use.\n * @returns {JSX.Element}\n */\nfunction FetchAndDisplay({endpoint, method = 'GET'}) {\n const [data, setData] = useState(null);\n const [error, setError] = useState(null);\n\n useEffect(() => {\n fetch(endpoint, {\n method,\n headers: {'Content-Type': 'application/json'},\n })\n .then((response) => {\n if (!response.ok) throw new Error(response.statusText);\n return response.json();\n })\n .then(setData)\n .catch((err) => setError(err.toString()));\n }, [endpoint, method]);\n\n return (\n \n
\n {method} {endpoint}\n \n {error &&
Error: {error}
}\n
{JSON.stringify(data, null, 2)} \n
}`.\n * If the response is missing `asset`, the update is skipped.\n *\n * @param {Object} updatedTranscription\n */\n const handleTranscriptionUpdate = (updatedTranscription) => {\n if (!updatedTranscription?.asset) {\n console.error(\n 'Missing asset on updatedTranscription:',\n updatedTranscription,\n );\n return;\n }\n\n setAssetData({\n ...updatedTranscription.asset,\n transcription: updatedTranscription,\n transcriptionStatus: updatedTranscription.asset.transcriptionStatus,\n });\n };\n\n if (error) return Error: {error}
;\n if (!assetData) return Loading asset data...
;\n\n return children({assetData, handleTranscriptionUpdate});\n}\n\n/**\n * Defines nested routes for a single asset view.\n *\n * Renders the split viewer by default and wires routes for supporting\n * actions like OCR, rollback, rollforward, submit and review.\n *\n * @param {Object} props\n * @param {Object} props.assetData\n * @param {Function} props.handleTranscriptionUpdate\n * @returns {JSX.Element}\n */\nfunction AssetRoutes({assetData, handleTranscriptionUpdate}) {\n return (\n <>\n \n \n \n );\n}\n\n/**\n * Renders navigation links for the current asset and optional\n * links for a specific transcription.\n *\n * @param {Object} props\n * @param {Object} props.assetData\n * @returns {JSX.Element|null}\n */\nfunction NavLinks({assetData}) {\n if (!assetData) return null;\n\n const currentAssetId = assetData.id;\n const transcriptionId = assetData.transcription?.id;\n\n return (\n \n \n );\n}\n\n/**\n * Fallback route for unknown paths.\n *\n * @returns {JSX.Element}\n */\nfunction NotFound() {\n return 404 Not Found ;\n}\n\n/**\n * Debug route: show transcriptions list payload for an asset.\n *\n * @returns {JSX.Element}\n */\nfunction Transcriptions() {\n const {assetId} = useParams();\n const endpoint = `/api/assets/${assetId}/transcriptions`;\n return \n \n \n {({assetData, handleTranscriptionUpdate}) => (\n \n {({assetData, handleTranscriptionUpdate}) => (\n \n \n );\n}\n"
},
{
"path": "frontend/src/ViewerSplit.jsx",
"content": "import React, {useLayoutEffect, useRef, useState} from 'react';\nimport Split from 'split.js';\n\nimport Editor from './editor/Editor';\nimport Viewer from './viewer/Viewer';\nimport OcrSection from './ocr/Section';\n\n/**\n * @typedef {Object} AssetData\n * @property {number} id\n * @property {string} imageUrl\n * @property {Object} [transcription]\n * Latest transcription object for the asset, or null if none.\n * @property {string} transcriptionStatus\n * One of the server statuses used to choose UI state.\n * @property {number} registeredContributors\n * @property {Array<[string,string]>} languages\n * Array of [isoCode, languageName] pairs used by OCR.\n * @property {boolean} undoAvailable\n * @property {boolean} redoAvailable\n */\n\n/**\n * Split-pane layout for the transcription UI.\n *\n * Renders the image viewer on one side and the editor on the other,\n * with a draggable gutter. The split direction and pane sizes persist\n * to localStorage. When the direction changes, the viewer is nudged to\n * re-fit the image.\n *\n * Local storage keys:\n * - \"transcription-split-sizes-vertical\" for vertical sizes\n * - \"transcription-split-sizes-horizontal\" for horizontal sizes\n * - \"transcription-split-direction\" for the active direction\n *\n * @param {{assetData: AssetData, onTranscriptionUpdate?: (t: Object) => void}} props\n * assetData: Data for the current asset.\n * onTranscriptionUpdate: Callback when a new transcription is saved\n * or loaded.\n */\nexport default function ViewerSplit({assetData, onTranscriptionUpdate}) {\n const contributeContainerRef = useRef(null);\n const editorColumnRef = useRef(null);\n\n const verticalKey = 'transcription-split-sizes-vertical';\n const horizontalKey = 'transcription-split-sizes-horizontal';\n const directionKey = 'transcription-split-direction';\n\n const [splitDirection, setSplitDirection] = useState(\n JSON.parse(localStorage.getItem(directionKey)) || 'h',\n );\n\n const [transcription, setTranscription] = useState(assetData.transcription);\n\n /**\n * Handle an updated transcription payload from child components.\n * Updates local state and forwards to the optional parent callback.\n *\n * @param {Object} updated\n */\n const handleTranscriptionUpdate = (updated) => {\n if (!updated?.text) {\n console.warn(\n 'handleTranscriptionUpdate called with malformed object:',\n updated,\n );\n return;\n }\n setTranscription(updated);\n if (onTranscriptionUpdate) onTranscriptionUpdate(updated);\n };\n\n /**\n * Update only the transcription text field as the user types.\n *\n * @param {string} newText\n */\n const handleTranscriptionTextChange = (newText) => {\n setTranscription((prev) => ({\n ...prev,\n text: newText,\n }));\n };\n\n /**\n * Read persisted Split.js sizes or return the provided defaults.\n *\n * @param {string} key\n * @param {number[]} defaultSizes\n * @returns {number[]}\n */\n const getSizes = (key, defaultSizes) => {\n const sizes = localStorage.getItem(key);\n return sizes ? JSON.parse(sizes) : defaultSizes;\n };\n\n /**\n * Persist pane sizes for the current direction.\n *\n * @param {number[]} sizes\n */\n const saveSizes = (sizes) => {\n const key = splitDirection === 'h' ? horizontalKey : verticalKey;\n localStorage.setItem(key, JSON.stringify(sizes));\n };\n\n /**\n * Persist the active split direction.\n *\n * @param {'h'|'v'} dir\n */\n const saveDirection = (dir) => {\n localStorage.setItem(directionKey, JSON.stringify(dir));\n };\n\n /**\n * Create or recreate the Split.js instance whenever direction changes.\n * Cleans up on unmount. Uses flex-basis so panes respect the gutter size.\n */\n useLayoutEffect(() => {\n const sizes =\n splitDirection === 'h'\n ? getSizes(horizontalKey, [50, 50])\n : getSizes(verticalKey, [50, 50]);\n\n const splitInstance = Split(['#viewer-column', '#editor-column'], {\n sizes,\n minSize: 100,\n gutterSize: 8,\n direction: splitDirection === 'h' ? 'horizontal' : 'vertical',\n elementStyle: (dimension, size, gutterSize) => ({\n flexBasis: `calc(${size}% - ${gutterSize}px)`,\n }),\n gutterStyle: (dimension, gutterSize) => ({\n flexBasis: `${gutterSize}px`,\n }),\n onDragEnd: saveSizes,\n });\n\n return () => {\n splitInstance.destroy();\n };\n }, [splitDirection]);\n\n /**\n * Toggle between horizontal and vertical layouts.\n * Saves direction then requests the OpenSeadragon viewer to re-fit.\n *\n * @param {'h'|'v'} dir\n */\n const handleToggle = (dir) => {\n if (dir !== splitDirection) {\n setSplitDirection(dir);\n saveDirection(dir);\n setTimeout(() => {\n if (window.seadragonViewer?.viewport) {\n window.seadragonViewer.viewport.zoomTo(1, undefined, true);\n }\n }, 10);\n }\n };\n\n return (\n \n
\n \n handleToggle('h')}\n onLayoutVertical={() => handleToggle('v')}\n />\n \n
\n ![\"{{](\"{{)
\n \n ![\"Carrie](\"{%)
\n