feat: upstream-downstream entity linking [FC-0076]

cms/djangoapps/contentstore/admin.py

@@ -13,6 +13,8 @@
 from cms.djangoapps.contentstore.models import (
     BackfillCourseTabsConfig,
     CleanStaleCertificateAvailabilityDatesConfig,
+    LearningContextLinksStatus,
+    PublishableEntityLink,
     VideoUploadConfig
 )
 from cms.djangoapps.contentstore.outlines_regenerate import CourseOutlineRegenerate
@@ -86,6 +88,71 @@ class CleanStaleCertificateAvailabilityDatesConfigAdmin(ConfigurationModelAdmin)
     pass
 
 
+@admin.register(PublishableEntityLink)
+class PublishableEntityLinkAdmin(admin.ModelAdmin):
+    """
+    PublishableEntityLink admin.
+    """
+    fields = (
+        "uuid",
+        "upstream_block",
+        "upstream_usage_key",
+        "upstream_context_key",
+        "downstream_usage_key",
+        "downstream_context_key",
+        "version_synced",
+        "version_declined",
+        "created",
+        "updated",
+    )
+    readonly_fields = fields
+    list_display = [
+        "upstream_block",
+        "upstream_usage_key",
+        "downstream_usage_key",
+        "version_synced",
+        "updated",
+    ]
+    search_fields = [
+        "upstream_usage_key",
+        "upstream_context_key",
+        "downstream_usage_key",
+        "downstream_context_key",
+    ]
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+
+@admin.register(LearningContextLinksStatus)
+class LearningContextLinksStatusAdmin(admin.ModelAdmin):
+    """
+    LearningContextLinksStatus admin.
+    """
+    fields = (
+        "context_key",
+        "status",
+        "created",
+        "updated",
+    )
+    readonly_fields = ("created", "updated")
+    list_display = (
+        "context_key",
+        "status",
+        "created",
+        "updated",
+    )
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+
 admin.site.register(BackfillCourseTabsConfig, ConfigurationModelAdmin)
 admin.site.register(VideoUploadConfig, ConfigurationModelAdmin)
 admin.site.register(CourseOutlineRegenerate, CourseOutlineRegenerateAdmin)

cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py

@@ -0,0 +1,94 @@
+"""
+Management command to recreate upstream-dowstream links in PublishableEntityLink for course(s).
+
+This command can be run for all the courses or for given list of courses.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import gettext as _
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import CourseKey
+
+from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
+
+from ...tasks import create_or_update_upstream_links
+
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Recreate links for course(s) in PublishableEntityLink table.
+
+    Examples:
+        # Recreate upstream links for two courses.
+        $ ./manage.py cms recreate_upstream_links --course course-v1:edX+DemoX.1+2014 \
+        --course course-v1:edX+DemoX.2+2015
+        # Force recreate upstream links for one or more courses including processed ones.
+        $ ./manage.py cms recreate_upstream_links --course course-v1:edX+DemoX.1+2014 \
+        --course course-v1:edX+DemoX.2+2015 --force
+        # Recreate upstream links for all courses.
+        $ ./manage.py cms recreate_upstream_links --all
+        # Force recreate links for all courses including completely processed ones.
+        $ ./manage.py cms recreate_upstream_links --all --force
+        # Delete all links and force recreate links for all courses
+        $ ./manage.py cms recreate_upstream_links --all --force --replace
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--course',
+            metavar=_('COURSE_KEY'),
+            action='append',
+            help=_('Recreate links for xblocks under given course keys. For eg. course-v1:edX+DemoX.1+2014'),
+            default=[],
+        )
+        parser.add_argument(
+            '--all',
+            action='store_true',
+            help=_(
+                'Recreate links for xblocks under all courses. NOTE: this can take long time depending'
+                ' on number of course and xblocks'
+            ),
+        )
+        parser.add_argument(
+            '--force',
+            action='store_true',
+            help=_('Recreate links even for completely processed courses.'),
+        )
+        parser.add_argument(
+            '--replace',
+            action='store_true',
+            help=_('Delete all and create links for given course(s).'),
+        )
+
+    def handle(self, *args, **options):
+        """
+        Handle command
+        """
+        courses = options['course']
+        should_process_all = options['all']
+        force = options['force']
+        replace = options['replace']
+        time_now = datetime.now(tz=timezone.utc)
+        if not courses and not should_process_all:
+            raise CommandError('Either --course or --all argument should be provided.')
+
+        if should_process_all and courses:
+            raise CommandError('Only one of --course or --all argument should be provided.')
+
+        if should_process_all:
+            courses = CourseOverview.get_all_course_keys()
+        for course in courses:
+            log.info(f"Start processing upstream->dowstream links in course: {course}")
+            try:
+                CourseKey.from_string(str(course))
+            except InvalidKeyError:
+                log.error(f"Invalid course key: {course}, skipping..")
+                continue
+            create_or_update_upstream_links.delay(str(course), force=force, replace=replace, created=time_now)

cms/djangoapps/contentstore/migrations/0009_learningcontextlinksstatus_publishableentitylink.py

@@ -0,0 +1,93 @@
+# Generated by Django 4.2.18 on 2025-02-05 05:33
+
+import uuid
+
+import django.db.models.deletion
+import opaque_keys.edx.django.models
+import openedx_learning.lib.fields
+import openedx_learning.lib.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('oel_publishing', '0002_alter_learningpackage_key_and_more'),
+        ('contentstore', '0008_cleanstalecertificateavailabilitydatesconfig'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LearningContextLinksStatus',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                (
+                    'context_key',
+                    opaque_keys.edx.django.models.CourseKeyField(
+                        help_text='Linking status for course context key', max_length=255, unique=True
+                    ),
+                ),
+                (
+                    'status',
+                    models.CharField(
+                        choices=[
+                            ('pending', 'Pending'),
+                            ('processing', 'Processing'),
+                            ('failed', 'Failed'),
+                            ('completed', 'Completed'),
+                        ],
+                        help_text='Status of links in given learning context/course.',
+                        max_length=20,
+                    ),
+                ),
+                ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])),
+                ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])),
+            ],
+            options={
+                'verbose_name': 'Learning Context Links status',
+                'verbose_name_plural': 'Learning Context Links status',
+            },
+        ),
+        migrations.CreateModel(
+            name='PublishableEntityLink',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')),
+                (
+                    'upstream_usage_key',
+                    opaque_keys.edx.django.models.UsageKeyField(
+                        help_text='Upstream block usage key, this value cannot be null and useful to track upstream library blocks that do not exist yet',
+                        max_length=255,
+                    ),
+                ),
+                (
+                    'upstream_context_key',
+                    openedx_learning.lib.fields.MultiCollationCharField(
+                        db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'},
+                        db_index=True,
+                        help_text='Upstream context key i.e., learning_package/library key',
+                        max_length=500,
+                    ),
+                ),
+                ('downstream_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255, unique=True)),
+                ('downstream_context_key', opaque_keys.edx.django.models.CourseKeyField(db_index=True, max_length=255)),
+                ('version_synced', models.IntegerField()),
+                ('version_declined', models.IntegerField(blank=True, null=True)),
+                ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])),
+                ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])),
+                (
+                    'upstream_block',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name='links',
+                        to='oel_publishing.publishableentity',
+                    ),
+                ),
+            ],
+            options={
+                'verbose_name': 'Publishable Entity Link',
+                'verbose_name_plural': 'Publishable Entity Links',
+            },
+        ),
+    ]