feanil/fix ulmo docs build

.readthedocs.yaml

@@ -10,6 +10,10 @@ sphinx:
 
 python:
   install:
+    # Need to install this to set the correct version of steuptools for now
+    # because it is needed by fs
+    # See https://github.com/openedx/openedx-platform/issues/38068 for details.
+  - requirements: "requirements/pip-tools.txt"
   - requirements: "requirements/edx/doc.txt"
   - method: pip
     path: .

Makefile

@@ -116,7 +116,7 @@ $(COMMON_CONSTRAINTS_TXT):
 	printf "$(COMMON_CONSTRAINTS_TEMP_COMMENT)" | cat - $(@) > temp && mv temp $(@)
 
 compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade
-compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *.in requirements to *.txt
+compile-requirements: pre-requirements ## Re-compile *.in requirements to *.txt
 	@# Bootstrapping: Rebuild pip and pip-tools first, and then install them
 	@# so that if there are any failures we'll know now, rather than the next
 	@# time someone tries to use the outputs.
@@ -139,7 +139,7 @@ compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *
 		export REBUILD=''; \
 	done
 
-upgrade:  ## update the pip requirements files to use the latest releases satisfying our constraints
+upgrade:  $(COMMON_CONSTRAINTS_TXT) ## update the pip requirements files to use the latest releases satisfying our constraints
 	$(MAKE) compile-requirements COMPILE_OPTS="--upgrade"
 
 upgrade-package: ## update just one package to the latest usable release

cms/djangoapps/cms_user_tasks/signals.py

@@ -12,6 +12,7 @@
 
 from cms.djangoapps.contentstore.toggles import bypass_olx_failure_enabled
 from cms.djangoapps.contentstore.utils import course_import_olx_validation_is_enabled
+from openedx.core.djangoapps.content_libraries.api import is_library_backup_task, is_library_restore_task
 
 from .tasks import send_task_complete_email
 
@@ -64,6 +65,28 @@ def get_olx_validation_from_artifact():
         if olx_artifact and not bypass_olx_failure_enabled():
             return olx_artifact.text
 
+    def should_skip_end_of_task_email(task_name) -> bool:
+        """
+        Studio tasks generally send an email when finished, but not always.
+
+        Some tasks can last many minutes, e.g. course import/export. For these
+        tasks, there is a high chance that the user has navigated away and will
+        want to check back in later. Yet email notification is unnecessary and
+        distracting for things like the Library restore task, which is
+        relatively quick and cannot be resumed (i.e. if you navigate away, you
+        have to upload again).
+
+        The task_name passed in will be lowercase.
+        """
+        # We currently have to pattern match on the name to differentiate
+        # between tasks. A better long term solution would be to add a separate
+        # task type identifier field to Django User Tasks.
+        return (
+            is_library_content_update(task_name) or
+            is_library_backup_task(task_name) or
+            is_library_restore_task(task_name)
+        )
+
     status = kwargs['status']
 
     # Only send email when the entire task is complete, should only send when
@@ -72,7 +95,7 @@ def get_olx_validation_from_artifact():
         task_name = status.name.lower()
 
         # Also suppress emails on library content XBlock updates (too much like spam)
-        if is_library_content_update(task_name):
+        if should_skip_end_of_task_email(task_name):
             LOGGER.info(f"Suppressing end-of-task email on task {task_name}")
             return
 

cms/djangoapps/contentstore/api/tests/test_validation.py

@@ -2,20 +2,25 @@
 Tests for the course import API views
 """
 
-
+import factory
 from datetime import datetime
+from django.conf import settings
 
+import ddt
 from django.test.utils import override_settings
 from django.urls import reverse
 from rest_framework import status
 from rest_framework.test import APITestCase
 from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
 from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory
 
+from common.djangoapps.course_modes.models import CourseMode
+from common.djangoapps.course_modes.tests.factories import CourseModeFactory
 from common.djangoapps.student.tests.factories import StaffFactory
 from common.djangoapps.student.tests.factories import UserFactory
 
 
+@ddt.ddt
 @override_settings(PROCTORING_BACKENDS={'DEFAULT': 'proctortrack', 'proctortrack': {}})
 class CourseValidationViewTest(SharedModuleStoreTestCase, APITestCase):
     """
@@ -82,39 +87,54 @@ def test_student_fails(self):
         resp = self.client.get(self.get_url(self.course_key))
         self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
-    def test_staff_succeeds(self):
-        self.client.login(username=self.staff.username, password=self.password)
-        resp = self.client.get(self.get_url(self.course_key), {'all': 'true'})
-        self.assertEqual(resp.status_code, status.HTTP_200_OK)
-        expected_data = {
-            'assignments': {
-                'total_number': 1,
-                'total_visible': 1,
-                'assignments_with_dates_before_start': [],
-                'assignments_with_dates_after_end': [],
-                'assignments_with_ora_dates_after_end': [],
-                'assignments_with_ora_dates_before_start': [],
-            },
-            'dates': {
-                'has_start_date': True,
-                'has_end_date': False,
-            },
-            'updates': {
-                'has_update': True,
-            },
-            'certificates': {
-                'is_enabled': False,
-                'is_activated': False,
-                'has_certificate': False,
-            },
-            'grades': {
-                'has_grading_policy': False,
-                'sum_of_weights': 1.0,
-            },
-            'proctoring': {
-                'needs_proctoring_escalation_email': True,
-                'has_proctoring_escalation_email': True,
-            },
-            'is_self_paced': True,
-        }
-        self.assertDictEqual(resp.data, expected_data)
+    @ddt.data(
+        (False, False),
+        (True, False),
+        (False, True),
+        (True, True),
+    )
+    @ddt.unpack
+    def test_staff_succeeds(self, certs_html_view, with_modes):
+        features = dict(settings.FEATURES, CERTIFICATES_HTML_VIEW=certs_html_view)
+        with override_settings(FEATURES=features):
+            if with_modes:
+                CourseModeFactory.create_batch(
+                    2,
+                    course_id=self.course.id,
+                    mode_slug=factory.Iterator([CourseMode.AUDIT, CourseMode.VERIFIED]),
+                )
+            self.client.login(username=self.staff.username, password=self.password)
+            resp = self.client.get(self.get_url(self.course_key), {'all': 'true'})
+            self.assertEqual(resp.status_code, status.HTTP_200_OK)
+            expected_data = {
+                'assignments': {
+                    'total_number': 1,
+                    'total_visible': 1,
+                    'assignments_with_dates_before_start': [],
+                    'assignments_with_dates_after_end': [],
+                    'assignments_with_ora_dates_after_end': [],
+                    'assignments_with_ora_dates_before_start': [],
+                },
+                'dates': {
+                    'has_start_date': True,
+                    'has_end_date': False,
+                },
+                'updates': {
+                    'has_update': True,
+                },
+                'certificates': {
+                    'is_enabled': with_modes,
+                    'is_activated': False,
+                    'has_certificate': False,
+                },
+                'grades': {
+                    'has_grading_policy': False,
+                    'sum_of_weights': 1.0,
+                },
+                'proctoring': {
+                    'needs_proctoring_escalation_email': True,
+                    'has_proctoring_escalation_email': True,
+                },
+                'is_self_paced': True,
+            }
+            self.assertDictEqual(resp.data, expected_data)

cms/djangoapps/contentstore/models.py

@@ -7,12 +7,12 @@
 
 from config_models.models import ConfigurationModel
 from django.db import models
-from django.db.models import QuerySet, OuterRef, Case, When, Exists, Value, ExpressionWrapper
-from django.db.models.fields import IntegerField, TextField, BooleanField
+from django.db.models import Case, Exists, ExpressionWrapper, OuterRef, Q, QuerySet, Value, When
+from django.db.models.fields import BooleanField, IntegerField, TextField
 from django.db.models.functions import Coalesce
 from django.db.models.lookups import GreaterThan
 from django.utils.translation import gettext_lazy as _
-from opaque_keys.edx.django.models import CourseKeyField, ContainerKeyField, UsageKeyField
+from opaque_keys.edx.django.models import ContainerKeyField, CourseKeyField, UsageKeyField
 from opaque_keys.edx.keys import CourseKey, UsageKey
 from opaque_keys.edx.locator import LibraryContainerLocator
 from openedx_learning.api.authoring import get_published_version
@@ -23,7 +23,6 @@
     manual_date_time_field,
 )
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -391,7 +390,7 @@ def filter_links(
             cls.objects.filter(**link_filter).select_related(*RELATED_FIELDS),
         )
         if ready_to_sync is not None:
-            result = result.filter(ready_to_sync=ready_to_sync)
+            result = result.filter(Q(ready_to_sync=ready_to_sync) | Q(ready_to_sync_from_children=ready_to_sync))
 
         # Handle top-level parents logic
         if use_top_level_parents:
@@ -436,6 +435,11 @@ def _annotate_query_with_ready_to_sync(cls, query_set: QuerySet["EntityLinkBase"
                     ),
                     then=1
                 ),
+                # If upstream block was deleted, set ready_to_sync = True
+                When(
+                    Q(upstream_container__publishable_entity__published__version__version_num__isnull=True),
+                    then=1
+                ),
                 default=0,
                 output_field=models.IntegerField()
             )
@@ -457,6 +461,11 @@ def _annotate_query_with_ready_to_sync(cls, query_set: QuerySet["EntityLinkBase"
                     ),
                     then=1
                 ),
+                # If upstream block was deleted, set ready_to_sync = True
+                When(
+                    Q(upstream_block__publishable_entity__published__version__version_num__isnull=True),
+                    then=1
+                ),
                 default=0,
                 output_field=models.IntegerField()
             )

cms/djangoapps/contentstore/rest_api/v1/serializers/home.py

@@ -28,26 +28,11 @@ class LibraryViewSerializer(serializers.Serializer):
     org = serializers.CharField()
     number = serializers.CharField()
     can_edit = serializers.BooleanField()
-    is_migrated = serializers.SerializerMethodField()
-    migrated_to_title = serializers.CharField(
-        source="migrations__target__title",
-        required=False
-    )
-    migrated_to_key = serializers.CharField(
-        source="migrations__target__key",
-        required=False
-    )
-    migrated_to_collection_key = serializers.CharField(
-        source="migrations__target_collection__key",
-        required=False
-    )
-    migrated_to_collection_title = serializers.CharField(
-        source="migrations__target_collection__title",
-        required=False
-    )
-
-    def get_is_migrated(self, obj):
-        return "migrations__target__key" in obj
+    is_migrated = serializers.BooleanField()
+    migrated_to_title = serializers.CharField(required=False)
+    migrated_to_key = serializers.CharField(required=False)
+    migrated_to_collection_key = serializers.CharField(required=False)
+    migrated_to_collection_title = serializers.CharField(required=False)
 
 
 class CourseHomeTabSerializer(serializers.Serializer):

cms/djangoapps/contentstore/rest_api/v1/views/home.py

@@ -236,7 +236,7 @@ def get(self, request: Request):
                 "number": "CPSPR",
                 "can_edit": true
                 }
-            ],        }
+            ],
         ```
         """
 

cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py

@@ -18,7 +18,6 @@
 from cms.djangoapps.contentstore.tests.utils import CourseTestCase
 from cms.djangoapps.modulestore_migrator import api as migrator_api
 from cms.djangoapps.modulestore_migrator.data import CompositionLevel, RepeatHandlingStrategy
-from cms.djangoapps.modulestore_migrator.tests.factories import ModulestoreSourceFactory
 from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory
 from openedx.core.djangoapps.content_libraries import api as lib_api
 
@@ -253,8 +252,9 @@ class HomePageLibrariesViewTest(LibraryTestCase):
 
     def setUp(self):
         super().setUp()
-        # Create an additional legacy library
+        # Create an two additional legacy libaries
         self.lib_key_1 = self._create_library(library="lib1")
+        self.lib_key_2 = self._create_library(library="lib2")
         self.organization = OrganizationFactory()
 
         # Create a new v2 library
@@ -269,7 +269,6 @@ def setUp(self):
         library = lib_api.ContentLibrary.objects.get(slug=self.lib_key_v2.slug)
         learning_package = library.learning_package
         # Create a migration source for the legacy library
-        self.source = ModulestoreSourceFactory(key=self.lib_key_1)
         self.url = reverse("cms.djangoapps.contentstore:v1:libraries")
         # Create a collection to migrate this library to
         collection_key = "test-collection"
@@ -280,20 +279,32 @@ def setUp(self):
             created_by=self.user.id,
         )
 
-        # Migrate self.lib_key_1 to self.lib_key_v2
+        # Migrate both lib_key_1 and lib_key_2 to v2
+        # Only make lib_key_1 a "forwarding" migration.
         migrator_api.start_migration_to_library(
             user=self.user,
-            source_key=self.source.key,
+            source_key=self.lib_key_1,
             target_library_key=self.lib_key_v2,
             target_collection_slug=collection_key,
-            composition_level=CompositionLevel.Component.value,
-            repeat_handling_strategy=RepeatHandlingStrategy.Skip.value,
+            composition_level=CompositionLevel.Component,
+            repeat_handling_strategy=RepeatHandlingStrategy.Skip,
+            preserve_url_slugs=True,
+            forward_source_to_target=True,
+        )
+        migrator_api.start_migration_to_library(
+            user=self.user,
+            source_key=self.lib_key_2,
+            target_library_key=self.lib_key_v2,
+            target_collection_slug=collection_key,
+            composition_level=CompositionLevel.Component,
+            repeat_handling_strategy=RepeatHandlingStrategy.Skip,
             preserve_url_slugs=True,
             forward_source_to_target=False,
         )
 
     def test_home_page_libraries_response(self):
-        """Check successful response content"""
+        """Check sucessful response content"""
+        self.maxDiff = None
         response = self.client.get(self.url)
 
         expected_response = {
@@ -322,6 +333,17 @@ def test_home_page_libraries_response(self):
                     'migrated_to_collection_key': 'test-collection',
                     'migrated_to_collection_title': 'Test Collection',
                 },
+                # Third library was migrated, but not with forwarding.
+                # So, it appears just like the unmigrated library.
+                {
+                    'display_name': 'Test Library',
+                    'library_key': 'library-v1:org+lib2',
+                    'url': '/library/library-v1:org+lib2',
+                    'org': 'org',
+                    'number': 'lib2',
+                    'can_edit': True,
+                    'is_migrated': False,
+                },
             ]
         }
 
@@ -366,6 +388,15 @@ def test_home_page_libraries_response(self):
                     'can_edit': True,
                     'is_migrated': False,
                 },
+                {
+                    'display_name': 'Test Library',
+                    'library_key': 'library-v1:org+lib2',
+                    'url': '/library/library-v1:org+lib2',
+                    'org': 'org',
+                    'number': 'lib2',
+                    'can_edit': True,
+                    'is_migrated': False,
+                },
             ],
         }