From 09373fb2a9bacdea99ed6fc218c28b8eb6b6886e Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Tue, 5 Nov 2024 17:03:58 -0500 Subject: [PATCH 01/27] add AccessLogExportTask class, refactored Project View exports to create csv files for access logs, and added tests --- ...ove_projectviewexporttask_data_and_more.py | 88 +++++++++ kpi/models/import_export_task.py | 41 ++-- kpi/tasks.py | 16 +- kpi/tests/test_access_logs_export_task.py | 124 ++++++++++++ ...roject_view_exports.py => data_exports.py} | 186 ++++++++++++------ 5 files changed, 386 insertions(+), 69 deletions(-) create mode 100644 kpi/migrations/0060_commonexporttask_remove_projectviewexporttask_data_and_more.py create mode 100644 kpi/tests/test_access_logs_export_task.py rename kpi/utils/{project_view_exports.py => data_exports.py} (52%) diff --git a/kpi/migrations/0060_commonexporttask_remove_projectviewexporttask_data_and_more.py b/kpi/migrations/0060_commonexporttask_remove_projectviewexporttask_data_and_more.py new file mode 100644 index 0000000000..395e5c6123 --- /dev/null +++ b/kpi/migrations/0060_commonexporttask_remove_projectviewexporttask_data_and_more.py @@ -0,0 +1,88 @@ +# Generated by Django 4.2.15 on 2024-11-05 15:47 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import kpi.fields.file +import kpi.fields.kpi_uid +import kpi.models.asset_file +import kpi.models.import_export_task +import private_storage.fields +import private_storage.storage.files + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('kpi', '0059_assetexportsettings_date_created_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='CommonExportTask', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('data', models.JSONField()), + ('messages', models.JSONField(default=dict)), + ('status', models.CharField(choices=[('created', 'created'), ('processing', 'processing'), ('error', 'error'), ('complete', 'complete')], default='created', max_length=32)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('result', private_storage.fields.PrivateFileField(max_length=380, storage=private_storage.storage.files.PrivateFileSystemStorage(), upload_to=kpi.models.import_export_task.export_upload_to)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.RemoveField( + model_name='projectviewexporttask', + name='data', + ), + migrations.RemoveField( + model_name='projectviewexporttask', + name='date_created', + ), + migrations.RemoveField( + model_name='projectviewexporttask', + name='id', + ), + migrations.RemoveField( + model_name='projectviewexporttask', + name='messages', + ), + migrations.RemoveField( + model_name='projectviewexporttask', + name='result', + ), + migrations.RemoveField( + model_name='projectviewexporttask', + name='status', + ), + migrations.RemoveField( + model_name='projectviewexporttask', + name='user', + ), + migrations.AlterField( + model_name='assetfile', + name='content', + field=kpi.fields.file.PrivateExtendedFileField(max_length=380, null=True, upload_to=kpi.models.asset_file.upload_to), + ), + migrations.CreateModel( + name='AccessLogExportTask', + fields=[ + ('commonexporttask_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='kpi.commonexporttask')), + ('uid', kpi.fields.kpi_uid.KpiUidField(_null=False, uid_prefix='ale')), + ('get_all_logs', models.BooleanField(default=False)), + ], + options={ + 'abstract': False, + }, + bases=('kpi.commonexporttask',), + ), + migrations.AddField( + model_name='projectviewexporttask', + name='commonexporttask_ptr', + field=models.OneToOneField(auto_created=True, default=None, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='kpi.commonexporttask'), + preserve_default=False, + ), + ] diff --git a/kpi/models/import_export_task.py b/kpi/models/import_export_task.py index 322f2cad76..8c1a4a712a 100644 --- a/kpi/models/import_export_task.py +++ b/kpi/models/import_export_task.py @@ -57,9 +57,9 @@ from kpi.exceptions import XlsFormatException from kpi.fields import KpiUidField from kpi.models import Asset +from kpi.utils.data_exports import create_data_export from kpi.utils.log import logging from kpi.utils.models import _load_library_content, create_assets, resolve_url_to_asset -from kpi.utils.project_view_exports import create_project_view_export from kpi.utils.rename_xls_sheet import ( ConflictSheetError, NoFromSheetError, @@ -469,27 +469,23 @@ def export_upload_to(self, filename): return posixpath.join(self.user.username, 'exports', filename) -class ProjectViewExportTask(ImportExportTask): - uid = KpiUidField(uid_prefix='pve') +class CommonExportTask(ImportExportTask): result = PrivateFileField(upload_to=export_upload_to, max_length=380) + def _get_export_details(self) -> tuple: + return self.data['type'], self.data['view'] + def _build_export_filename( self, export_type: str, username: str, view: str ) -> str: time = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ') return f'{export_type}-{username}-view_{view}-{time}.csv' - def _run_task(self, messages: list) -> None: - export_type = self.data['type'] - view = self.data['view'] - - filename = self._build_export_filename( - export_type, self.user.username, view - ) + def _run_task_base(self, messages: list, buff) -> None: + export_type, view = self._get_export_details() + filename = self._build_export_filename(export_type, self.user.username, view) absolute_filepath = self.get_absolute_filepath(filename) - buff = create_project_view_export(export_type, self.user.username, view) - with self.result.storage.open(absolute_filepath, 'wb') as output_file: output_file.write(buff.read().encode()) @@ -502,6 +498,27 @@ def delete(self, *args, **kwargs) -> None: super().delete(*args, **kwargs) +class AccessLogExportTask(CommonExportTask): + uid = KpiUidField(uid_prefix='ale') + get_all_logs = models.BooleanField(default=False) + + def _run_task(self, messages: list) -> None: + if self.get_all_logs and not self.user.is_superuser: + raise PermissionError('Only superusers can export all access logs.') + + export_type, view = self._get_export_details() + buff = create_data_export(export_type, self.user.username, self.uid, self.get_all_logs) + self._run_task_base(messages, buff) + +class ProjectViewExportTask(CommonExportTask): + uid = KpiUidField(uid_prefix='pve') + + def _run_task(self, messages: list) -> None: + export_type, view = self._get_export_details() + buff = create_data_export(export_type, self.user.username, view, False) + self._run_task_base(messages, buff) + + class ExportTaskBase(ImportExportTask): """ An (asynchronous) submission data export job. The instantiator must set the diff --git a/kpi/tasks.py b/kpi/tasks.py index 050b73108a..fc9123d70e 100644 --- a/kpi/tasks.py +++ b/kpi/tasks.py @@ -45,8 +45,22 @@ def export_task_in_background( 'Regards,\n' 'KoboToolbox' ) + subject_map = { + 'AccessLog': 'Access Log Report Complete', + 'ProjectView': 'Project View Report Complete', + } + + subject = next( + ( + subject + for key, subject in subject_map.items() + if key in export_task_name + ), + 'Report Complete', + ) + mail.send_mail( - subject='Project View Report Complete', + subject=subject, message=msg, from_email=constance.config.SUPPORT_EMAIL, recipient_list=[user.email], diff --git a/kpi/tests/test_access_logs_export_task.py b/kpi/tests/test_access_logs_export_task.py new file mode 100644 index 0000000000..d01a6fb91a --- /dev/null +++ b/kpi/tests/test_access_logs_export_task.py @@ -0,0 +1,124 @@ +import csv +import os + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.test import TestCase + +from kobo.apps.audit_log.models import AccessLog +from kpi.models.import_export_task import AccessLogExportTask + +User = get_user_model() + + +class AccessLogExportTaskTests(TestCase): + + def setUp(self): + self.user = User.objects.create_user( + username='testuser', email='testuser@example.com', password='password' + ) + self.superuser = User.objects.create_superuser( + username='superuser', email='superuser@example.com', password='password' + ) + + def create_export_task(self, user, get_all_logs=True): + return AccessLogExportTask.objects.create( + user=user, + get_all_logs=get_all_logs, + data={'view': 'assets', 'type': 'access_logs_export'}, + ) + + def test_task_initialization(self): + task = self.create_export_task(self.user, get_all_logs=False) + self.assertIsInstance(task, AccessLogExportTask) + self.assertFalse(task.get_all_logs) + + def test_get_all_logs_superuser(self): + task = self.create_export_task(self.superuser) + self.assertTrue(task.get_all_logs) + + def test_get_all_logs_non_superuser(self): + task = self.create_export_task(self.user) + + with self.assertRaises(PermissionError) as context: + task._run_task([]) + + self.assertEqual( + str(context.exception), 'Only superusers can export all access logs.' + ) + + def test_run_task_creates_csv(self): + task = self.create_export_task(self.superuser) + task.run() + + self.assertIsNotNone(task.result, 'The task.result should not be None.') + expected_pattern = ( + rf'{self.superuser.username}/exports/access_logs_export-' + rf'{self.superuser.username}-view_assets-' + r'\d{4}-\d{2}-\d{2}T\d{6}Z\.csv' + ) + + self.assertRegex( + task.result.name, + expected_pattern, + 'The task.result file name format is incorrect.', + ) + self.assertTrue( + os.path.exists(task.result.path), + f'The file at {task.result.path} should exist.', + ) + + def test_csv_content_structure(self): + log = AccessLog.objects.create( + user=self.user, + metadata={ + 'auth_type': 'test_auth', + 'source': 'test_source', + 'ip_address': '127.0.0.1', + 'initial_user_username': 'initial_superuser', + 'initial_user_uid': 'initial_superuser_uid', + 'authorized_app_name': 'test_app', + }, + date_created='2024-11-05T12:00:00Z', + ) + task = self.create_export_task(self.superuser) + task.run() + + with open(task.result.path, mode='r', encoding='utf-8') as csv_file: + reader = csv.DictReader(csv_file) + rows = list(reader) + + expected_headers = [ + 'user_url', + 'user_uid', + 'username', + 'auth_type', + 'date_created', + 'source', + 'ip_address', + 'initial_superusername', + 'initial_superuseruid', + 'authorized_application', + 'other_details', + ] + self.assertListEqual(expected_headers, reader.fieldnames) + + first_row = rows[0] + expected_user_url = ( + f'{settings.KOBOFORM_URL}/api/v2/users/{self.user.username}' + ) + + self.assertEqual(first_row['user_url'], expected_user_url) + self.assertEqual(first_row['user_uid'], log.user_uid) + self.assertEqual(first_row['username'], self.user.username) + self.assertEqual(first_row['auth_type'], 'test_auth') + self.assertEqual(first_row['source'], 'test_source') + self.assertEqual(first_row['ip_address'], '127.0.0.1') + self.assertEqual(first_row['initial_superusername'], 'initial_superuser') + self.assertEqual(first_row['initial_superuseruid'], 'initial_superuser_uid') + self.assertEqual(first_row['authorized_application'], 'test_app') + self.assertIsNotNone(first_row['other_details']) + + def tearDown(self): + AccessLogExportTask.objects.all().delete() + User.objects.all().delete() diff --git a/kpi/utils/project_view_exports.py b/kpi/utils/data_exports.py similarity index 52% rename from kpi/utils/project_view_exports.py rename to kpi/utils/data_exports.py index d8ab5e1643..c11d905d0a 100644 --- a/kpi/utils/project_view_exports.py +++ b/kpi/utils/data_exports.py @@ -6,9 +6,11 @@ from typing import Union from django.conf import settings -from django.db.models import Count, F, Q +from django.db.models import CharField, Count, F, Q, Value +from django.db.models.functions import Concat from django.db.models.query import QuerySet +from kobo.apps.audit_log.models import AccessLog from kobo.apps.kobo_auth.shortcuts import User from kobo.apps.openrosa.apps.logger.models.xform import XForm from kpi.constants import ASSET_TYPE_SURVEY @@ -66,6 +68,19 @@ 'instagram', 'metadata', ) +ACCESS_LOGS_EXPORT_FIELDS = ( + 'user_url', + 'user_uid', + 'username', + 'auth_type', + 'date_created', + 'source', + 'ip_address', + 'initial_superusername', + 'initial_superuseruid', + 'authorized_application', + 'other_details', +) CONFIG = { 'assets': { 'queryset': Asset.objects.filter(asset_type=ASSET_TYPE_SURVEY), @@ -79,9 +94,73 @@ 'key': METADATA, 'columns': USER_FIELDS + METADATA_FIELDS, }, + 'access_logs_export': { + 'queryset': AccessLog.objects.all(), + 'key': 'metadata', + 'columns': ACCESS_LOGS_EXPORT_FIELDS, + }, } +def create_data_export( + export_type: str, username: str, uid: str, get_all_logs: bool = False +) -> StringIO: + config = CONFIG[export_type] + + # For access logs, modify the queryset based on the user's superuser status + if export_type == 'access_logs_export': + queryset = ( + config['queryset'].all() + if get_all_logs + else config['queryset'].filter(user__username=username) + ) + else: + region_for_view = get_region_for_view(uid) + q = get_q(region_for_view, export_type) + queryset = config['queryset'].filter(q) + + data = get_data(queryset, export_type) + if export_type == 'access_logs_export': + # Make sure other_details only contains metadata that has not already been + # accessed + accessed_metadata_fields = [ + 'auth_type', + 'source', + 'ip_address', + 'initial_user_username', + 'initial_user_uid', + 'auth_app_name', + ] + for row in data: + row['other_details'] = filter_remaining_metadata( + row, accessed_metadata_fields + ) + buff = StringIO() + writer = csv.writer(buff) + writer.writerow(config['columns']) + for row in data: + items = row.pop(config['key'], {}) or {} + flatten_settings_inplace(items) + row.update(items) + # submission counts come from kobocat database and therefore need to be + # appended manually rather than through queries + if export_type == 'assets': + row['submission_count'] = get_submission_count(row['form_id']) + flat_row = [get_row_value(row, col) for col in config['columns']] + writer.writerow(flat_row) + + buff.seek(0) + return buff + + +def filter_remaining_metadata(row, accessed_fields): + metadata = row['other_details'] + if metadata is not None: + return { + key: value for key, value in metadata.items() if key not in accessed_fields + } + + def flatten_settings_inplace(settings: dict) -> None: for k, v in settings.items(): if isinstance(v, list) and v: @@ -98,12 +177,48 @@ def flatten_settings_inplace(settings: dict) -> None: settings[k] = '' -def get_row_value(row: dict, col: str) -> Union[str, int, float, bool, None]: - val = row.get(col, '') - # remove any new lines from text - if isinstance(val, str): - val = val.replace('\n', '') - return val +def get_data(filtered_queryset: QuerySet, export_type: str) -> QuerySet: + if export_type == 'assets': + vals = ASSET_FIELDS + (SETTINGS,) + return ( + filtered_queryset.annotate( + owner__name=F('owner__extra_details__data__name'), + owner__organization=F('owner__extra_details__data__organization'), + form_id=F('_deployment_data__backend_response__formid'), + ) + .values(*vals) + .order_by('id') + ) + elif export_type == 'users': + vals = USER_FIELDS + (METADATA,) + return ( + filtered_queryset.exclude(pk=settings.ANONYMOUS_USER_ID) + .annotate( + mfa_is_active=F('mfa_methods__is_active'), + metadata=F('extra_details__data'), + asset_count=Count('assets'), + ) + .values(*vals) + .order_by('id') + ) + elif export_type == 'access_logs_export': + user_url = Concat( + Value(f'{settings.KOBOFORM_URL}/api/v2/users/'), + F('user__username'), + output_field=CharField(), + ) + + return filtered_queryset.annotate( + user_url=user_url, + username=F('user__username'), + auth_type=F('metadata__auth_type'), + source=F('metadata__source'), + ip_address=F('metadata__ip_address'), + initial_superusername=F('metadata__initial_user_username'), + initial_superuseruid=F('metadata__initial_user_uid'), + authorized_application=F('metadata__authorized_app_name'), + other_details=F('metadata'), + ).values(*ACCESS_LOGS_EXPORT_FIELDS) def get_q(countries: list[str], export_type: str) -> QuerySet: @@ -115,6 +230,14 @@ def get_q(countries: list[str], export_type: str) -> QuerySet: return Q(**{q_term: countries}) +def get_row_value(row: dict, col: str) -> Union[str, int, float, bool, None]: + val = row.get(col, '') + # remove any new lines from text + if isinstance(val, str): + val = val.replace('\n', '') + return val + + def get_submission_count(xform_id: int) -> int: result = XForm.objects.values('num_of_submissions').filter(pk=xform_id).first() @@ -123,52 +246,3 @@ def get_submission_count(xform_id: int) -> int: return 0 return result['num_of_submissions'] - - -def get_data(filtered_queryset: QuerySet, export_type: str) -> QuerySet: - if export_type == 'assets': - vals = ASSET_FIELDS + (SETTINGS,) - data = filtered_queryset.annotate( - owner__name=F('owner__extra_details__data__name'), - owner__organization=F('owner__extra_details__data__organization'), - form_id=F('_deployment_data__backend_response__formid'), - ) - else: - vals = USER_FIELDS + (METADATA,) - data = filtered_queryset.exclude( - pk=settings.ANONYMOUS_USER_ID - ).annotate( - mfa_is_active=F('mfa_methods__is_active'), - metadata=F('extra_details__data'), - asset_count=Count('assets'), - ) - - return data.values(*vals).order_by('id') - - -def create_project_view_export( - export_type: str, username: str, uid: str -) -> StringIO: - config = CONFIG[export_type] - region_for_view = get_region_for_view(uid) - - q = get_q(region_for_view, export_type) - filtered_queryset = config['queryset'].filter(q) - data = get_data(filtered_queryset, export_type) - - buff = StringIO() - writer = csv.writer(buff) - writer.writerow(config['columns']) - for row in data: - items = row.pop(config['key'], {}) or {} - flatten_settings_inplace(items) - row.update(items) - # submission counts come from kobocat database and therefore need to be - # appended manually rather than through queries - if export_type == 'assets': - row['submission_count'] = get_submission_count(row['form_id']) - flat_row = [get_row_value(row, col) for col in config['columns']] - writer.writerow(flat_row) - - buff.seek(0) - return buff From 4079ea05a8eed915ad74143a43c7be665b70e6a8 Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Thu, 14 Nov 2024 16:22:09 -0500 Subject: [PATCH 02/27] revert to before Submissions name change --- .github/CODEOWNERS | 45 +- README.md | 4 +- dependencies/pip/dev_requirements.txt | 4 +- dependencies/pip/requirements.in | 2 +- dependencies/pip/requirements.txt | 4 +- hub/admin/extend_user.py | 43 +- hub/models/extra_user_detail.py | 5 +- hub/static/admin/css/inline_as_fieldset.css | 3 + jsapp/js/account/accountSidebar.tsx | 18 +- .../account/add-ons/addOnList.component.tsx | 323 ++++---- .../js/account/add-ons/addOnList.module.scss | 117 ++- .../add-ons/oneTimeAddOnRow.component.tsx | 205 +++++ .../add-ons/updateBadge.component..tsx | 62 ++ .../billingContextProvider.component.tsx | 7 +- .../requireOrgOwner.component.tsx | 33 - .../account/plans/billingButton.module.scss | 1 + .../account/plans/planContainer.component.tsx | 8 +- .../js/account/plans/useDisplayPrice.hook.tsx | 6 + jsapp/js/account/routes.tsx | 57 +- jsapp/js/account/stripe.api.ts | 73 +- jsapp/js/account/stripe.types.ts | 34 +- jsapp/js/account/stripe.utils.ts | 52 +- .../oneTimeAddOnList.component.tsx | 69 ++ .../oneTimeAddOnList.module.scss | 21 + .../oneTimeAddOnUsageModal.component.tsx | 105 +++ .../oneTimeAddOnUsageModal.module.scss | 57 ++ jsapp/js/account/usage/usage.component.tsx | 151 +++- jsapp/js/account/usage/usageContainer.tsx | 178 ++--- jsapp/js/account/useOneTimeAddonList.hook.ts | 37 + jsapp/js/api.endpoints.ts | 1 + jsapp/js/components/common/button.scss | 3 +- .../header/mainHeader.component.tsx | 7 +- .../components/header/mainHeader.module.scss | 6 + .../header/organizationBadge.module.scss | 1 - jsapp/js/components/modals/koboModal.scss | 1 - ...rojectOwnershipTransferModalWithBanner.tsx | 75 ++ .../projectTransferInviteBanner.module.scss | 19 + .../projectTransferInviteBanner.tsx | 64 ++ .../special/koboAccessibleSelect.module.scss | 16 +- .../special/koboAccessibleSelect.tsx | 105 ++- .../usageLimits/useExceedingLimits.hook.ts | 30 +- jsapp/js/constants.ts | 2 +- jsapp/js/dataInterface.ts | 6 +- jsapp/js/envStore.ts | 6 +- jsapp/js/featureFlags.ts | 3 +- jsapp/js/projects/customViewRoute.tsx | 162 +--- jsapp/js/projects/customViewStore.ts | 18 +- jsapp/js/projects/myProjectsRoute.module.scss | 39 - jsapp/js/projects/myProjectsRoute.tsx | 247 +----- jsapp/js/projects/projectViews.module.scss | 43 +- jsapp/js/projects/projectViews/constants.ts | 2 + .../projectViews/projectsFieldsSelector.tsx | 9 +- .../projectViews/projectsFilter.module.scss | 4 + .../projects/projectViews/projectsFilter.tsx | 11 +- .../projectViews/projectsFilterEditor.tsx | 13 +- .../js/projects/projectViews/viewSwitcher.tsx | 29 +- .../bulkActions/bulkDeletePrompt.tsx | 13 +- .../projectsTable/projectBulkActions.tsx | 13 +- .../projectsTable/projectQuickActions.tsx | 19 +- .../projects/projectsTable/projectsTable.tsx | 15 +- .../projectsTable/projectsTableHeader.tsx | 15 +- .../projectsTableRow.module.scss | 1 + .../projectsTable/projectsTableRow.tsx | 29 +- .../sortableProjectColumnHeader.tsx | 14 +- jsapp/js/projects/universalProjectsRoute.tsx | 205 +++++ .../validateOrgPermissions.component.tsx | 46 ++ ...paginatedQueryUniversalTable.component.tsx | 2 +- kobo/apps/audit_log/audit_actions.py | 10 + kobo/apps/audit_log/base_views.py | 26 +- kobo/apps/audit_log/models.py | 142 +++- kobo/apps/audit_log/signals.py | 12 +- kobo/apps/audit_log/tests/test_models.py | 213 +++++- .../tests/test_project_history_logs.py | 512 ++++++++++++- kobo/apps/hook/tests/hook_test_case.py | 126 +--- kobo/apps/hook/utils/tests/__init__.py | 0 kobo/apps/hook/utils/tests/mixins.py | 119 +++ kobo/apps/hook/views/v2/hook.py | 12 +- kobo/apps/kobo_auth/models.py | 14 +- .../tests/fixtures/Transportation Form.xml | 326 ++++---- .../viewsets/test_xform_submission_api.py | 6 +- .../api/tests/viewsets/test_xform_viewset.py | 1 + .../apps/api/viewsets/xform_submission_api.py | 9 +- .../commands/populate_submission_counters.py | 21 +- .../update_attachment_storage_bytes.py | 19 +- ..._populate_daily_xform_counters_for_year.py | 34 +- .../0030_backfill_lost_monthly_counters.py | 68 -- .../0031_remove_null_user_daily_counters.py | 10 - ...032_alter_daily_submission_counter_user.py | 2 +- .../0038_add_mongo_uuid_field_to_xform.py | 22 + .../migrations/0039_populate_counters.py | 120 +++ .../openrosa/apps/logger/models/instance.py | 7 +- .../apps/openrosa/apps/logger/models/xform.py | 74 +- .../0017_userprofile_submissions_suspended.py | 18 + .../openrosa/apps/main/models/user_profile.py | 1 + .../apps/openrosa/apps/main/service_health.py | 49 -- .../transportation/transportation.xml | 329 ++++---- kobo/apps/openrosa/apps/main/urls.py | 259 ++++--- .../apps/viewer/models/data_dictionary.py | 4 +- .../apps/viewer/models/parsed_instance.py | 13 +- kobo/apps/openrosa/libs/filters.py | 56 +- kobo/apps/openrosa/libs/utils/logger_tools.py | 4 +- kobo/apps/organizations/admin.py | 64 -- kobo/apps/organizations/admin/__init__.py | 7 + kobo/apps/organizations/admin/organization.py | 90 +++ .../admin/organization_invite.py | 8 + .../organizations/admin/organization_owner.py | 24 + .../organizations/admin/organization_user.py | 156 ++++ kobo/apps/organizations/constants.py | 8 +- kobo/apps/organizations/exceptions.py | 2 + kobo/apps/organizations/forms.py | 24 + .../0006_update_organization_name.py | 10 +- kobo/apps/organizations/models.py | 147 +++- kobo/apps/organizations/permissions.py | 28 +- kobo/apps/organizations/serializers.py | 7 +- kobo/apps/organizations/tasks.py | 30 + ..._organization.py => test_organizations.py} | 27 +- .../tests/test_organizations_api.py | 711 +++++++++++++++++- .../tests/test_organizations_model.py | 22 + kobo/apps/organizations/types.py | 3 + kobo/apps/organizations/utils.py | 29 +- kobo/apps/organizations/views.py | 83 +- .../0003_create_proxy_and_add_invite_type.py | 53 ++ kobo/apps/project_ownership/models/invite.py | 153 +++- .../apps/project_ownership/models/transfer.py | 30 +- .../project_ownership/serializers/invite.py | 151 +--- .../tests/api/v2/test_api.py | 68 +- kobo/apps/project_ownership/utils.py | 77 +- .../service_health/test_service_health.py | 10 - kobo/apps/service_health/views.py | 36 +- kobo/apps/stripe/admin.py | 59 ++ kobo/apps/stripe/constants.py | 11 + kobo/apps/stripe/migrations/0001_initial.py | 102 +++ kobo/apps/stripe/migrations/__init__.py | 0 kobo/apps/stripe/models.py | 254 +++++++ kobo/apps/stripe/serializers.py | 21 +- .../templates/admin/add-ons/change_list.html | 8 + .../stripe/tests/test_customer_portal_api.py | 18 +- .../stripe/tests/test_one_time_addons_api.py | 188 ++++- .../stripe/tests/test_organization_usage.py | 53 +- .../stripe/tests/test_subscription_api.py | 4 +- kobo/apps/stripe/utils.py | 90 ++- kobo/apps/stripe/views.py | 62 +- .../integrations/google/google_transcribe.py | 2 + .../tests/test_submission_extras_api_post.py | 15 +- kobo/apps/trackers/tests/test_utils.py | 121 +++ kobo/apps/trackers/utils.py | 69 +- kobo/apps/trash_bin/utils.py | 4 +- kobo/settings/base.py | 10 + kpi/backends.py | 7 +- kpi/db_routers.py | 9 +- kpi/deployment_backends/openrosa_backend.py | 35 +- kpi/filters.py | 40 +- kpi/fixtures/asset_with_settings_and_qa.json | 10 +- kpi/fixtures/conflicting_versions.json | 71 +- ...ove_projectviewexporttask_data_and_more.py | 76 +- ...nousexport_submissionssynchronousexport.py | 19 + ...rename_exporttask_submissionsexporttask.py | 19 + kpi/mixins/object_permission.py | 58 +- kpi/models/__init__.py | 6 +- kpi/models/asset_file.py | 18 +- kpi/models/import_export_task.py | 65 +- kpi/models/paired_data.py | 4 + kpi/paginators.py | 2 +- kpi/permissions.py | 31 +- kpi/serializers/v1/export_task.py | 6 +- kpi/serializers/v2/asset.py | 29 +- kpi/serializers/v2/export_task.py | 11 +- kpi/serializers/v2/service_usage.py | 4 + kpi/tasks.py | 5 +- kpi/tests/api/test_api_environment.py | 30 +- kpi/tests/api/v1/test_api_assets.py | 10 +- kpi/tests/api/v1/test_api_exports.py | 8 +- .../test_api_asset_permission_assignment.py | 3 +- kpi/tests/api/v2/test_api_assets.py | 125 +-- kpi/tests/api/v2/test_api_collections.py | 83 +- kpi/tests/api/v2/test_api_exports.py | 4 +- kpi/tests/api/v2/test_api_imports.py | 88 ++- kpi/tests/api/v2/test_api_permissions.py | 15 +- kpi/tests/api/v2/test_api_service_usage.py | 8 +- kpi/tests/api/v2/test_api_submissions.py | 172 +---- kpi/tests/base_test_case.py | 87 ++- kpi/tests/kpi_test_case.py | 55 -- kpi/tests/test_asset_snapshots.py | 2 +- kpi/tests/test_cache_utils.py | 55 ++ ...t_mock_data_conflicting_version_exports.py | 9 +- kpi/tests/test_mock_data_exports.py | 48 +- kpi/tests/test_organization.py | 49 -- kpi/tests/test_permissions.py | 48 +- kpi/tests/test_usage_calculator.py | 42 +- kpi/tests/utils/mixins.py | 336 +++++++++ kpi/utils/cache.py | 106 ++- kpi/utils/data_exports.py | 14 +- kpi/utils/django_orm_helper.py | 32 +- kpi/utils/project_views.py | 2 + kpi/utils/pyxform_compatibility.py | 38 + kpi/utils/usage_calculator.py | 66 +- kpi/views/environment.py | 3 +- kpi/views/v1/export_task.py | 30 +- kpi/views/v1/import_task.py | 6 + kpi/views/v2/asset.py | 58 +- kpi/views/v2/asset_export_settings.py | 4 +- kpi/views/v2/asset_file.py | 24 +- kpi/views/v2/asset_snapshot.py | 66 +- kpi/views/v2/export_task.py | 11 +- kpi/views/v2/import_task.py | 12 +- kpi/views/v2/paired_data.py | 19 +- pip-compile.sh | 6 +- 207 files changed, 8031 insertions(+), 3073 deletions(-) create mode 100644 hub/static/admin/css/inline_as_fieldset.css create mode 100644 jsapp/js/account/add-ons/oneTimeAddOnRow.component.tsx create mode 100644 jsapp/js/account/add-ons/updateBadge.component..tsx delete mode 100644 jsapp/js/account/organizations/requireOrgOwner.component.tsx create mode 100644 jsapp/js/account/usage/one-time-add-on-usage-modal/one-time-add-on-list/oneTimeAddOnList.component.tsx create mode 100644 jsapp/js/account/usage/one-time-add-on-usage-modal/one-time-add-on-list/oneTimeAddOnList.module.scss create mode 100644 jsapp/js/account/usage/one-time-add-on-usage-modal/oneTimeAddOnUsageModal.component.tsx create mode 100644 jsapp/js/account/usage/one-time-add-on-usage-modal/oneTimeAddOnUsageModal.module.scss create mode 100644 jsapp/js/account/useOneTimeAddonList.hook.ts create mode 100644 jsapp/js/components/permissions/transferProjects/projectOwnershipTransferModalWithBanner.tsx create mode 100644 jsapp/js/components/permissions/transferProjects/projectTransferInviteBanner.module.scss create mode 100644 jsapp/js/components/permissions/transferProjects/projectTransferInviteBanner.tsx delete mode 100644 jsapp/js/projects/myProjectsRoute.module.scss create mode 100644 jsapp/js/projects/universalProjectsRoute.tsx create mode 100644 jsapp/js/router/validateOrgPermissions.component.tsx create mode 100644 kobo/apps/hook/utils/tests/__init__.py create mode 100644 kobo/apps/hook/utils/tests/mixins.py create mode 100644 kobo/apps/openrosa/apps/logger/migrations/0038_add_mongo_uuid_field_to_xform.py create mode 100644 kobo/apps/openrosa/apps/logger/migrations/0039_populate_counters.py create mode 100644 kobo/apps/openrosa/apps/main/migrations/0017_userprofile_submissions_suspended.py delete mode 100644 kobo/apps/openrosa/apps/main/service_health.py delete mode 100644 kobo/apps/organizations/admin.py create mode 100644 kobo/apps/organizations/admin/__init__.py create mode 100644 kobo/apps/organizations/admin/organization.py create mode 100644 kobo/apps/organizations/admin/organization_invite.py create mode 100644 kobo/apps/organizations/admin/organization_owner.py create mode 100644 kobo/apps/organizations/admin/organization_user.py create mode 100644 kobo/apps/organizations/exceptions.py create mode 100644 kobo/apps/organizations/forms.py create mode 100644 kobo/apps/organizations/tasks.py rename kobo/apps/organizations/tests/{test_organization.py => test_organizations.py} (79%) create mode 100644 kobo/apps/organizations/tests/test_organizations_model.py create mode 100644 kobo/apps/organizations/types.py create mode 100644 kobo/apps/project_ownership/migrations/0003_create_proxy_and_add_invite_type.py create mode 100644 kobo/apps/stripe/admin.py create mode 100644 kobo/apps/stripe/migrations/0001_initial.py create mode 100644 kobo/apps/stripe/migrations/__init__.py create mode 100644 kobo/apps/stripe/models.py create mode 100644 kobo/apps/stripe/templates/admin/add-ons/change_list.html create mode 100644 kobo/apps/trackers/tests/test_utils.py create mode 100644 kpi/migrations/0061_rename_synchronousexport_submissionssynchronousexport.py create mode 100644 kpi/migrations/0062_rename_exporttask_submissionsexporttask.py create mode 100644 kpi/tests/test_cache_utils.py delete mode 100644 kpi/tests/test_organization.py create mode 100644 kpi/tests/utils/mixins.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4d94ea64e3..106021ed79 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -36,20 +36,22 @@ /jsapp/ @magicznyleszek /patches/ @p2edwards /static/ @magicznyleszek -/webpack/ @magicznyleszek -/.babelrc.json @magicznyleszek -/.browserlistrc @magicznyleszek -/.eslintignore @magicznyleszek -/.eslintrc.js @magicznyleszek -/.node-version @magicznyleszek -/.nvmrc @magicznyleszek -/.prettierrc.js @magicznyleszek -/.stylelintrc.js @magicznyleszek -/.swcrc @magicznyleszek -/.coffeelnt.json @magicznyleszek -/package-lock.json @magicznyleszek -/package.json @magicznyleszek -/tsconfig.json @magicznyleszek +/static/js/ @magicznyleszek @p2edwards +/test/ @magicznyleszek @p2edwards +/webpack/ @magicznyleszek @p2edwards +/.babelrc.json @magicznyleszek @p2edwards +/.browserslistrc @magicznyleszek @p2edwards +/.eslintignore @magicznyleszek @p2edwards +/.eslintrc.js @magicznyleszek @p2edwards +/.node-version @magicznyleszek @p2edwards +/.nvmrc @magicznyleszek @p2edwards +/.prettierrc.js @magicznyleszek @p2edwards +/.stylelintrc.js @magicznyleszek @p2edwards +/.swcrc @magicznyleszek @p2edwards +/.coffeelint.json @magicznyleszek @p2edwards +/package-lock.json @magicznyleszek @p2edwards +/package.json @magicznyleszek @p2edwards +/tsconfig.json @magicznyleszek @p2edwards # Billing /jsapp/js/account/ @jamesrkiger @@ -64,11 +66,12 @@ # Default owner /dependencies/ @jnm @noliveleger /hub/ @jnm @noliveleger +/hub/tests/ @jnm @noliveleger /kobo/ @jnm @noliveleger /kobo/apps/audit_log/ @rgraber /kobo/apps/subsequences/ @Guitlle /kpi/ @jnm @noliveleger -/test/ @jnm @noliveleger +/kpi/tests/ @jnm @noliveleger /.coveragerc @jnm /.dockerignore @jnm @noliveleger /format-python.sh @noliveleger @@ -76,6 +79,12 @@ /pip-compile.sh @jnm @noliveleger /pyproject.toml @jnm @noliveleger +# Django Static Templates, HTML, JS, CSS +/kobo/apps/accounts/templates/ @magicznyleszek @p2edwards @jnm @noliveleger +/kpi/templates/ @magicznyleszek @p2edwards @jnm @noliveleger +/kpi/static/css/ @magicznyleszek @p2edwards +_registration.scss @magicznyleszek @p2edwards + @@ -84,7 +93,9 @@ /docker/ @bufke @jnm @noliveleger /scripts/ @bufke @jnm @noliveleger +/scripts/*.js @magicznyleszek @p2edwards /.github/ @bufke @jnm @noliveleger @magicznyleszek +/.github/workflows/npm-test.yml @bufke @jnm @noliveleger @magicznyleszek @p2edwards /.gitlab-ci.yml @bufke /Dockerfile @bufke @jnm @noliveleger @@ -95,8 +106,8 @@ #### Documentation section /CONTRIBUTING.md @Akuukis @magicznyleszek @noliveleger /README.md @Akuukis @jnm @magicznyleszek @noliveleger -/.github/CODEOWNERS.md @Akuukis @jnm @noliveleger @magicznyleszek -/.github/FUNDING.md @Akuukis @jnm @noliveleger @magicznyleszek +/.github/CODEOWNERS @Akuukis @jnm @noliveleger @magicznyleszek +/.github/FUNDING.yml @Akuukis @jnm @noliveleger @magicznyleszek /.github/ISSUE_TEMPLATE.md @Akuukis @jnm @noliveleger @magicznyleszek /.github/PULL_REQUEST_TEMPLATE.md @Akuukis @jnm @noliveleger @magicznyleszek diff --git a/README.md b/README.md index fbbd76336c..86ae437103 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # KPI -[![Python Build Status](https://github.com/kobotoolbox/kpi/workflows/pytest/badge.svg?branch=main)](https://github.com/kobotoolbox/kpi/actions?query=workflow%3Apytest+branch%3Amain) +[![Python Build Status](https://github.com/kobotoolbox/kpi/actions/workflows/pytest.yml/badge.svg?branch=main)](https://github.com/kobotoolbox/kpi/actions?query=workflow%3Apytest+branch%3Amain) [![Python Coverage Status](https://coveralls.io/repos/github/kobotoolbox/kpi/badge.svg?branch=main)](https://coveralls.io/github/kobotoolbox/kpi?branch=main) -[![JavaScript Build Status](https://github.com/kobotoolbox/kpi/workflows/npm-test/badge.svg?branch=main)](https://github.com/kobotoolbox/kpi/actions?query=workflow%3Anpm-test+branch%3Amain) +[![JavaScript Build Status](https://github.com/kobotoolbox/kpi/actions/workflows/npm-test.yml/badge.svg?branch=main)](https://github.com/kobotoolbox/kpi/actions?query=workflow%3Anpm-test+branch%3Amain) For production always use a specific release branch, `main` branch may include breaking changes. Run `git branch -rl 'origin/release/*'` to list release branches and then switch to a release branch of your choice. diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index 13c4b905f7..1df8bbb9f1 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -414,7 +414,7 @@ oauthlib==3.2.2 # -r dependencies/pip/requirements.in # django-oauth-toolkit # requests-oauthlib -openpyxl==3.0.9 +openpyxl==3.1.3 # via # -r dependencies/pip/requirements.in # pyxform @@ -537,7 +537,7 @@ pytz==2024.1 # via # flower # pandas -pyxform==1.9.0 +pyxform==2.2.0 # via # -r dependencies/pip/requirements.in # formpack diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 5db53d0e5e..968f5b6ba8 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -75,7 +75,7 @@ openpyxl psycopg pymongo python-dateutil -pyxform==1.9.0 +pyxform==2.2.0 requests regex responses diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 41e5731fa0..76175a5dc5 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -336,7 +336,7 @@ oauthlib==3.2.2 # -r dependencies/pip/requirements.in # django-oauth-toolkit # requests-oauthlib -openpyxl==3.0.9 +openpyxl==3.1.3 # via # -r dependencies/pip/requirements.in # pyxform @@ -412,7 +412,7 @@ pytz==2024.1 # via # flower # pandas -pyxform==1.9.0 +pyxform==2.2.0 # via # -r dependencies/pip/requirements.in # formpack diff --git a/hub/admin/extend_user.py b/hub/admin/extend_user.py index 47116f5ede..b962f2a1ad 100644 --- a/hub/admin/extend_user.py +++ b/hub/admin/extend_user.py @@ -25,6 +25,7 @@ from kobo.apps.trash_bin.models.account import AccountTrash from kobo.apps.trash_bin.utils import move_to_trash from kpi.models.asset import AssetDeploymentStatus + from .filters import UserAdvancedSearchFilter from .mixins import AdvancedSearchMixin @@ -87,8 +88,10 @@ class OrgInline(admin.StackedInline): 'organization', 'is_admin', ] + can_delete = False + # Override H2 style to make inline section like other fieldsets + classes = ('no-upper',) raw_id_fields = ('user', 'organization') - readonly_fields = settings.STRIPE_ENABLED and ('active_subscription_status',) or [] def active_subscription_status(self, obj): if settings.STRIPE_ENABLED: @@ -98,6 +101,12 @@ def active_subscription_status(self, obj): else 'None' ) + def get_readonly_fields(self, request, obj=None): + readonly_fields = ['organization', 'is_admin'] + if settings.STRIPE_ENABLED: + readonly_fields.append('active_subscription_status') + return readonly_fields + def has_add_permission(self, request, obj=OrganizationUser): return False @@ -158,6 +167,9 @@ class ExtendedUserAdmin(AdvancedSearchMixin, UserAdmin): ) actions = ['remove', 'delete'] + class Media: + css = {'all': ('admin/css/inline_as_fieldset.css',)} + @admin.action(description='Remove selected users (delete everything but their username)') def remove(self, request, queryset, **kwargs): """ @@ -235,8 +247,9 @@ def get_queryset(self, request): ) def get_search_results(self, request, queryset, search_term): - if request.path != '/admin/auth/user/': + queryset = self._filter_queryset(request, queryset) + # If search comes from autocomplete field, use parent class method return super(UserAdmin, self).get_search_results( request, queryset, search_term @@ -261,6 +274,32 @@ def monthly_submission_count(self, obj): ).aggregate(counter=Sum('counter')) return instances.get('counter') + def _filter_queryset(self, request, queryset): + auto_complete = request.path == '/admin/autocomplete/' + app_label = request.GET.get('app_label') + model_name = request.GET.get('model_name') + + if ( + auto_complete + and app_label == 'organizations' + and model_name == 'organizationuser' + ): + return self._filter_queryset_for_organization_user(queryset) + + return queryset + + def _filter_queryset_for_organization_user(self, queryset): + """ + Displays only users whose organization has a single member. + """ + return ( + queryset.annotate( + user_count=Count('organizations_organization__organization_users') + ) + .filter(user_count__lte=1) + .order_by('username') + ) + def _remove_or_delete( self, request, diff --git a/hub/models/extra_user_detail.py b/hub/models/extra_user_detail.py index 012a28b676..95d27d650b 100644 --- a/hub/models/extra_user_detail.py +++ b/hub/models/extra_user_detail.py @@ -78,5 +78,6 @@ def _sync_org_name(self): except (KeyError, AttributeError): organization_name = None - user_organization.name = organization_name - user_organization.save(update_fields=['name']) + if organization_name: + user_organization.name = organization_name + user_organization.save(update_fields=['name']) diff --git a/hub/static/admin/css/inline_as_fieldset.css b/hub/static/admin/css/inline_as_fieldset.css new file mode 100644 index 0000000000..fc35ae7805 --- /dev/null +++ b/hub/static/admin/css/inline_as_fieldset.css @@ -0,0 +1,3 @@ +.no-upper h2 { + text-transform: unset; +} diff --git a/jsapp/js/account/accountSidebar.tsx b/jsapp/js/account/accountSidebar.tsx index a8ef94ca45..41aa3a54ba 100644 --- a/jsapp/js/account/accountSidebar.tsx +++ b/jsapp/js/account/accountSidebar.tsx @@ -48,10 +48,6 @@ function AccountSidebar() { setShowPlans(true); }, [subscriptionStore.isInitialised]); - const showAddOnsLink = useMemo(() => { - return !subscriptionStore.planResponse.length; - }, [subscriptionStore.isInitialised]); - return (