diff --git a/osf/management/commands/remove_duplicate_notification_subscriptions.py b/osf/management/commands/remove_duplicate_notification_subscriptions.py deleted file mode 100644 index 2c7b7775748..00000000000 --- a/osf/management/commands/remove_duplicate_notification_subscriptions.py +++ /dev/null @@ -1,70 +0,0 @@ -from django.core.management.base import BaseCommand -from django.db import transaction -from django.db.models import OuterRef, Exists - -from osf.models import NotificationSubscription - - -class Command(BaseCommand): - help = ( - 'Remove duplicate NotificationSubscription records, keeping only the highest-id record: ' - 'Default uniqueness: (user, content_type, object_id, notification_type, is_digest); ' - 'Optional uniqueness with --exclude-is-digest: (user, content_type, object_id, notification_type).' - ) - - def add_arguments(self, parser): - parser.add_argument( - '--dry', - action='store_true', - help='Show how many rows would be deleted without deleting anything.', - ) - parser.add_argument( - '--exclude-is-digest', - action='store_true', - default=False, - help='Whether to exclude _is_digest field in unique_together') - - def handle(self, *args, **options): - - self.stdout.write('Finding duplicate NotificationSubscription records…') - - if options['exclude_is_digest']: - to_remove = NotificationSubscription.objects.filter( - Exists( - NotificationSubscription.objects.filter( - user_id=OuterRef('user_id'), - content_type_id=OuterRef('content_type_id'), - object_id=OuterRef('object_id'), - notification_type_id=OuterRef('notification_type_id'), - id__gt=OuterRef('id'), # keep most recent record - ) - ) - ) - else: - to_remove = NotificationSubscription.objects.filter( - Exists( - NotificationSubscription.objects.filter( - user_id=OuterRef('user_id'), - content_type_id=OuterRef('content_type_id'), - object_id=OuterRef('object_id'), - notification_type_id=OuterRef('notification_type_id'), - _is_digest=OuterRef('_is_digest'), - id__gt=OuterRef('id'), # keep most recent record - ) - ) - ) - - count = to_remove.count() - self.stdout.write(f"Duplicates to remove: {count}") - - if count == 0: - self.stdout.write(self.style.SUCCESS('No duplicates found.')) - - if options['dry']: - self.stdout.write(self.style.WARNING('Dry run enabled — no records were deleted.')) - return - - if count > 0: - with transaction.atomic(): - deleted, _ = to_remove.delete() - self.stdout.write(self.style.SUCCESS(f"Successfully removed {deleted} duplicate records.")) diff --git a/scripts/remove_after_use/merge_notification_subscription_provider_ct.py b/scripts/remove_after_use/merge_notification_subscription_provider_ct.py index 50da93b2669..58a2dce03f9 100644 --- a/scripts/remove_after_use/merge_notification_subscription_provider_ct.py +++ b/scripts/remove_after_use/merge_notification_subscription_provider_ct.py @@ -7,6 +7,11 @@ from django.contrib.contenttypes.models import ContentType from framework.celery_tasks import app as celery_app from osf.models import NotificationSubscription +from django.db.models import Exists, OuterRef +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + @celery_app.task(name='scripts.remove_after_use.merge_notification_subscription_provider_ct') @@ -19,13 +24,37 @@ def merge_notification_subscription_provider_ct(): ContentType.objects.get_by_natural_key('osf', 'registrationprovider'), ContentType.objects.get_by_natural_key('osf', 'collectionprovider'), ] - subscriptions = NotificationSubscription.objects.filter( - content_type__in=provider_ct_list - ) - subscriptions.update( + + provider_ct_ids = [ct.id for ct in provider_ct_list] + + abstract_provider_ct_qs = NotificationSubscription.objects.filter( content_type=abstract_provider_ct ) + duplicates = NotificationSubscription.objects.filter( + content_type_id__in=provider_ct_ids + ).annotate( + abstract_exists=Exists( + abstract_provider_ct_qs.filter( + notification_type_id=OuterRef('notification_type_id'), + user_id=OuterRef('user_id'), + object_id=OuterRef('object_id'), + _is_digest=OuterRef('_is_digest'), + ) + ) + ).filter(abstract_exists=True) + + # delete rows that would conflict + logger.info(f'Deleted {duplicates.count()} duplicate NotificationSubscription rows with provider content types.') + duplicates.delete() + + # update remaining rows + update_qs = NotificationSubscription.objects.filter( + content_type_id__in=provider_ct_ids + ) + logger.info(f'Updated {update_qs.count()} NotificationSubscription rows to use abstract provider content type.') + update_qs.update(content_type=abstract_provider_ct) + if __name__ == '__main__': merge_notification_subscription_provider_ct.delay() diff --git a/scripts/remove_after_use/populate_notification_subscriptions_node_file_updated.py b/scripts/remove_after_use/populate_notification_subscriptions_node_file_updated.py deleted file mode 100644 index 61625ce6b1f..00000000000 --- a/scripts/remove_after_use/populate_notification_subscriptions_node_file_updated.py +++ /dev/null @@ -1,128 +0,0 @@ -import django -django.setup() - -from website.app import init_app -init_app(routes=False) - -from datetime import datetime -from framework.celery_tasks import app as celery_app -from django.contrib.contenttypes.models import ContentType -from django.db.models import Count, F, OuterRef, Subquery, IntegerField, CharField -from django.db.models.functions import Cast, Coalesce -from osf.models import Node, NotificationSubscription, NotificationTypeEnum - - -@celery_app.task(name='scripts.remove_after_use.populate_notification_subscriptions_node_file_updated') -def populate_notification_subscriptions_node_file_updated(batch_size: int = 1000): - print('---Starting NODE_FILE_UPDATED subscriptions population script----') - global_start = datetime.now() - - node_file_nt = NotificationTypeEnum.NODE_FILE_UPDATED - - node_ct = ContentType.objects.get_for_model(Node) - - node_notifications_sq = ( - NotificationSubscription.objects.filter( - content_type=node_ct, - notification_type=node_file_nt.instance, - object_id=Cast(OuterRef('pk'), CharField()), - ).values( - 'object_id' - ).annotate( - cnt=Count('id') - ).values('cnt')[:1] - ) - - nodes_qs = ( - Node.objects - .filter(is_deleted=False) - .annotate( - contributors_count=Count('_contributors', distinct=True), - notifications_count=Coalesce( - Subquery( - node_notifications_sq, - output_field=IntegerField(), - ), - 0 - ), - ).exclude(contributors_count=F('notifications_count')) - ).iterator(chunk_size=batch_size) - - items_to_create = [] - total_created = 0 - batch_start = datetime.now() - count_nodes = 0 - count_contributors = 0 - for node in nodes_qs: - count_nodes += 1 - for contributor in node.contributors.all(): - count_contributors += 1 - items_to_create.append( - NotificationSubscription( - notification_type=node_file_nt.instance, - user=contributor, - content_type=node_ct, - object_id=node.id, - _is_digest=True, - message_frequency='none', - ) - ) - if len(items_to_create) >= batch_size: - print(f'Creating batch of {len(items_to_create)} subscriptions...') - try: - NotificationSubscription.objects.bulk_create( - items_to_create, - batch_size=batch_size, - ignore_conflicts=True, - ) - total_created += len(items_to_create) - items_to_create = [] - except Exception as exeption: - print(f"Error during bulk_create: {exeption}") - continue - finally: - items_to_create.clear() - batch_end = datetime.now() - print(f'Batch took {batch_end - batch_start}') - - if count_contributors % batch_size == 0: - print(f'Processed {count_nodes} nodes with {count_contributors} contributors, created {total_created} subscriptions') - batch_start = datetime.now() - - if items_to_create: - final_batch_start = datetime.now() - print(f'Creating final batch of {len(items_to_create)} subscriptions...') - try: - NotificationSubscription.objects.bulk_create( - items_to_create, - batch_size=batch_size, - ignore_conflicts=True, - ) - total_created += len(items_to_create) - except Exception as exeption: - print(f"Error during bulk_create: {exeption}") - final_batch_end = datetime.now() - print(f'Final batch took {final_batch_end - final_batch_start}') - - global_end = datetime.now() - print(f'Total time for NODE_FILE_UPDATED subscription population: {global_end - global_start}') - print(f'Created {total_created} subscriptions.') - print('----Creation finished----') - -@celery_app.task(name='scripts.remove_after_use.update_notification_subscriptions_node_file_updated') -def update_notification_subscriptions_node_file_updated(): - print('---Starting NODE_FILE_UPDATED subscriptions update script----') - - node_file_nt = NotificationTypeEnum.NODE_FILE_UPDATED - - updated_start = datetime.now() - updated = ( - NotificationSubscription.objects.filter( - notification_type__name=node_file_nt, - _is_digest=False, - ) - .update(_is_digest=True) - ) - updated_end = datetime.now() - print(f'Updated {updated} subscriptions. Took time: {updated_end - updated_start}') - print('Update finished.') diff --git a/scripts/remove_after_use/populate_notification_subscriptions_user_global_file_updated.py b/scripts/remove_after_use/populate_notification_subscriptions_user_global_file_updated.py deleted file mode 100644 index 651143d6f8a..00000000000 --- a/scripts/remove_after_use/populate_notification_subscriptions_user_global_file_updated.py +++ /dev/null @@ -1,111 +0,0 @@ -import django -django.setup() - -from website.app import init_app -init_app(routes=False) - -from django.utils import timezone -from dateutil.relativedelta import relativedelta -from datetime import datetime -from framework.celery_tasks import app as celery_app -from django.contrib.contenttypes.models import ContentType -from osf.models import OSFUser, NotificationSubscription, NotificationTypeEnum - -@celery_app.task(name='scripts.remove_after_use.populate_notification_subscriptions_user_global_file_updated') -def populate_notification_subscriptions_user_global_file_updated(per_last_years: int | None= None, batch_size: int = 1000): - print('---Starting USER_FILE_UPDATED subscriptions population script----') - global_start = datetime.now() - - user_file_updated_nt = NotificationTypeEnum.USER_FILE_UPDATED - user_ct = ContentType.objects.get_for_model(OSFUser) - if per_last_years: - from_date = timezone.now() - relativedelta(years=per_last_years) - user_qs = (OSFUser.objects - .filter(date_last_login__gte=from_date) - .exclude(subscriptions__notification_type__name=user_file_updated_nt) - .distinct('id') - .order_by('id') - .iterator(chunk_size=batch_size) - ) - else: - user_qs = (OSFUser.objects - .exclude(subscriptions__notification_type__name=user_file_updated_nt) - .distinct('id') - .order_by('id') - .iterator(chunk_size=batch_size) - ) - - items_to_create = [] - total_created = 0 - - batch_start = datetime.now() - for count, user in enumerate(user_qs, 1): - items_to_create.append( - NotificationSubscription( - notification_type=user_file_updated_nt.instance, - user=user, - content_type=user_ct, - object_id=user.id, - _is_digest=True, - message_frequency='none', - ) - ) - if len(items_to_create) >= batch_size: - print(f'Creating batch of {len(items_to_create)} subscriptions...') - try: - NotificationSubscription.objects.bulk_create( - items_to_create, - batch_size=batch_size, - ignore_conflicts=True, - ) - total_created += len(items_to_create) - except Exception as e: - print(f'Error during bulk_create: {e}') - finally: - items_to_create.clear() - batch_end = datetime.now() - print(f'Batch took {batch_end - batch_start}') - - if count % batch_size == 0: - print(f'Processed {count}, created {total_created}') - batch_start = datetime.now() - - if items_to_create: - final_batch_start = datetime.now() - print(f'Creating final batch of {len(items_to_create)} subscriptions...') - try: - NotificationSubscription.objects.bulk_create( - items_to_create, - batch_size=batch_size, - ignore_conflicts=True, - ) - total_created += len(items_to_create) - except Exception as e: - print(f'Error during bulk_create: {e}') - final_batch_end = datetime.now() - print(f'Final batch took {final_batch_end - final_batch_start}') - - global_end = datetime.now() - print(f'Total time for USER_FILE_UPDATED subscription population: {global_end - global_start}') - print(f'Created {total_created} subscriptions.') - print('----Creation finished----') - -@celery_app.task(name='scripts.remove_after_use.update_notification_subscriptions_user_global_file_updated') -def update_notification_subscriptions_user_global_file_updated(): - print('---Starting USER_FILE_UPDATED subscriptions updating script----') - - user_file_updated_nt = NotificationTypeEnum.USER_FILE_UPDATED - - update_start = datetime.now() - updated = ( - NotificationSubscription.objects - .filter( - notification_type__name=user_file_updated_nt, - _is_digest=False, - ) - .update(_is_digest=True) - ) - update_end = datetime.now() - - print(f'Updated {updated} subscriptions. Took time: {update_end - update_start}') - print('Update finished.') diff --git a/scripts/remove_after_use/populate_notification_subscriptions_user_global_reviews.py b/scripts/remove_after_use/populate_notification_subscriptions_user_global_reviews.py deleted file mode 100644 index edfca287d0d..00000000000 --- a/scripts/remove_after_use/populate_notification_subscriptions_user_global_reviews.py +++ /dev/null @@ -1,104 +0,0 @@ -import django -django.setup() - -from website.app import init_app -init_app(routes=False) - -from django.utils import timezone -from dateutil.relativedelta import relativedelta -from datetime import datetime -from framework.celery_tasks import app as celery_app -from django.contrib.contenttypes.models import ContentType -from osf.models import OSFUser, NotificationSubscription, NotificationTypeEnum - - -@celery_app.task(name='scripts.remove_after_use.populate_notification_subscriptions_user_global_reviews') -def populate_notification_subscriptions_user_global_reviews(per_last_years: int | None = None, batch_size: int = 1000): - print('---Starting REVIEWS_SUBMISSION_STATUS subscriptions population script----') - global_start = datetime.now() - - review_nt = NotificationTypeEnum.REVIEWS_SUBMISSION_STATUS - user_ct = ContentType.objects.get_for_model(OSFUser) - if per_last_years: - from_date = timezone.now() - relativedelta(years=per_last_years) - user_qs = OSFUser.objects.filter(date_last_login__gte=from_date).exclude( - subscriptions__notification_type__name=review_nt.instance - ).distinct('id') - else: - user_qs = OSFUser.objects.exclude( - subscriptions__notification_type__name=review_nt.instance - ).distinct('id') - - items_to_create = [] - total_created = 0 - - batch_start = datetime.now() - for count, user in enumerate(user_qs, 1): - items_to_create.append( - NotificationSubscription( - notification_type=review_nt.instance, - user=user, - content_type=user_ct, - object_id=user.id, - _is_digest=True, - message_frequency='none', - ) - ) - if len(items_to_create) >= batch_size: - print(f'Creating batch of {len(items_to_create)} subscriptions...') - try: - NotificationSubscription.objects.bulk_create( - items_to_create, - batch_size=batch_size, - ignore_conflicts=True, - ) - total_created += len(items_to_create) - except Exception as e: - print(f'Error during bulk_create: {e}') - finally: - items_to_create.clear() - batch_end = datetime.now() - print(f'Batch took {batch_end - batch_start}') - - if count % batch_size == 0: - print(f'Processed {count}, created {total_created}') - batch_start = datetime.now() - - if items_to_create: - final_batch_start = datetime.now() - print(f'Creating final batch of {len(items_to_create)} subscriptions...') - try: - NotificationSubscription.objects.bulk_create( - items_to_create, - batch_size=batch_size, - ignore_conflicts=True, - ) - total_created += len(items_to_create) - except Exception as e: - print(f'Error during bulk_create: {e}') - final_batch_end = datetime.now() - print(f'Final batch took {final_batch_end - final_batch_start}') - - global_end = datetime.now() - print(f'Total time for REVIEWS_SUBMISSION_STATUS subscription population: {global_end - global_start}') - print(f'Created {total_created} subscriptions.') - print('----Creation finished----') - -@celery_app.task(name='scripts.remove_after_use.update_notification_subscriptions_user_global_reviews') -def update_notification_subscriptions_user_global_reviews(): - print('---Starting REVIEWS_SUBMISSION_STATUS subscriptions updating script----') - - review_nt = NotificationTypeEnum.REVIEWS_SUBMISSION_STATUS - - updated_start = datetime.now() - updated = ( - NotificationSubscription.objects.filter( - notification_type__name=review_nt, - _is_digest=False, - ) - .update(_is_digest=True) - ) - updated_end = datetime.now() - - print(f'Updated {updated} subscriptions. Took time: {updated_end - updated_start}') - print('Update finished.') diff --git a/website/routes.py b/website/routes.py index d7d6cf3d9bc..7110aaa7e77 100644 --- a/website/routes.py +++ b/website/routes.py @@ -1711,25 +1711,6 @@ def make_url_map(app): json_renderer, ), - # Legacy v1 API for notifications, which is no longer used by Angular/Post-NR - # Rule( - # '/subscriptions/', - # 'get', - # notification_views.get_subscriptions, - # json_renderer, - # ), - - # Legacy v1 API for notifications, which is no longer used by Angular/Post-NR - # Rule( - # [ - # '/project//subscriptions/', - # '/project//node//subscriptions/' - # ], - # 'get', - # notification_views.get_node_subscriptions, - # json_renderer, - # ), - Rule( [ '/project//tree/', @@ -1740,14 +1721,6 @@ def make_url_map(app): json_renderer, ), - # Legacy v1 API for notifications, which is no longer used by Angular/Post-NR - # Rule( - # '/subscriptions/', - # 'post', - # notification_views.configure_subscription, - # json_renderer, - # ), - Rule( [ '/project//settings/addons/', diff --git a/website/settings/defaults.py b/website/settings/defaults.py index 79562d7771c..82217585a52 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -436,9 +436,6 @@ class CeleryConfig: 'scripts.populate_new_and_noteworthy_projects', 'website.search.elastic_search', 'scripts.generate_sitemap', - 'scripts.remove_after_use.populate_notification_subscriptions_node_file_updated', - 'scripts.remove_after_use.populate_notification_subscriptions_user_global_file_updated', - 'scripts.remove_after_use.populate_notification_subscriptions_user_global_reviews', 'osf.management.commands.clear_expired_sessions', 'osf.management.commands.delete_withdrawn_or_failed_registration_files', 'osf.management.commands.migrate_pagecounter_data', @@ -568,9 +565,6 @@ class CeleryConfig: 'scripts.approve_embargo_terminations', 'scripts.triggered_mails', 'scripts.generate_sitemap', - 'scripts.remove_after_use.populate_notification_subscriptions_node_file_updated', - 'scripts.remove_after_use.populate_notification_subscriptions_user_global_file_updated', - 'scripts.remove_after_use.populate_notification_subscriptions_user_global_reviews', 'scripts.premigrate_created_modified', 'scripts.add_missing_identifiers_to_preprints', 'osf.management.commands.clear_expired_sessions',