diff --git a/cms/djangoapps/modulestore_migrator/api/read_api.py b/cms/djangoapps/modulestore_migrator/api/read_api.py index 757e1bf55a63..97d8a78c3951 100644 --- a/cms/djangoapps/modulestore_migrator/api/read_api.py +++ b/cms/djangoapps/modulestore_migrator/api/read_api.py @@ -5,18 +5,27 @@ import typing as t from uuid import UUID +from django.conf import settings from opaque_keys.edx.keys import UsageKey from opaque_keys.edx.locator import ( LibraryLocatorV2, LibraryUsageLocatorV2, LibraryContainerLocator ) -from openedx_learning.api.authoring import get_draft_version +from openedx_learning.api.authoring import get_draft_version, get_all_drafts from openedx_learning.api.authoring_models import ( PublishableEntityVersion, PublishableEntity, DraftChangeLogRecord ) +from xblock.plugin import PluginMissingError from openedx.core.djangoapps.content_libraries.api import ( - library_component_usage_key, library_container_locator + library_component_usage_key, library_container_locator, + validate_can_add_block_to_library, BlockLimitReachedError, + IncompatibleTypesError, LibraryBlockAlreadyExists, + ContentLibrary +) +from openedx.core.djangoapps.content.search.api import ( + fetch_block_types, + get_all_blocks_from_context, ) from ..data import ( @@ -32,6 +41,7 @@ 'get_forwarding_for_blocks', 'get_migrations', 'get_migration_blocks', + 'preview_migration', ) @@ -242,3 +252,120 @@ def _block_migration_success( target_title=target_title, target_version_num=target_version_num, ) + + +def preview_migration(source_key: SourceContextKey, target_key: LibraryLocatorV2): + """ + Returns a summary preview of the migration given a source key and a target key + on this form: + + ``` + { + "state": "partial", + "unsupported_blocks": 4, + "unsupported_percentage": 25, + "blocks_limit": 1000, + "total_blocks": 20, + "total_components": 10, + "sections": 2, + "subsections": 3, + "units": 5, + } + ``` + + List of states: + - 'success': The migration can be carried out in its entirety + - 'partial': The migration will be partial, because there are unsupported blocks. + - 'block_limit_reached': The migration cannot be performed because the block limit per library has been reached. + + This runs Meilisiearch queries to speed up the response, as it's a summary/analysis. + The decision has been made not to run a "migration" for each analysis to obtain this summary. + + TODO: For now, the repeat_handling_strategy is not taken into account. This can be taken into + account for a more advanced summary. + """ + # Get all containers and components from the source key + blocks = get_all_blocks_from_context(str(source_key), ["block_type", "block_id"]) + + unsupported_blocks = [] + total_blocks = 0 + total_components = 0 + sections = 0 + subsections = 0 + units = 0 + blocks_limit = settings.MAX_BLOCKS_PER_CONTENT_LIBRARY + + # Builds the summary: counts every container and verify if each component can be added to the library + for block in blocks: + block_type = block["block_type"] + block_id = block["block_id"] + total_blocks += 1 + if block_type not in ['chapter', 'sequential', 'vertical']: + total_components += 1 + try: + validate_can_add_block_to_library( + target_key, + block_type, + block_id, + ) + except BlockLimitReachedError: + return { + "state": "block_limit_reached", + "unsupported_blocks": 0, + "unsupported_percentage": 0, + "blocks_limit": blocks_limit, + "total_blocks": 0, + "total_components": 0, + "sections": 0, + "subsections": 0, + "units": 0, + } + except (IncompatibleTypesError, PluginMissingError): + unsupported_blocks.append(block["usage_key"]) + except LibraryBlockAlreadyExists: + # Skip this validation, The block may be repeated in the library, but that's not a bad thing. + pass + elif block_type == "chapter": + sections += 1 + elif block_type == "sequential": + subsections += 1 + elif block_type == "vertical": + units += 1 + + # Gets the count of children of unsupported blocks + quoted_keys = ','.join(f'"{key}"' for key in unsupported_blocks) + unsupportedBlocksChildren = fetch_block_types( + [ + f'context_key = "{source_key}"', + f'breadcrumbs.usage_key IN [{quoted_keys}]' + ], + ) + # Final unsupported blocks count + # The unsupported children are subtracted from the totals since they have already been counted in the first query. + unsupported_blocks_count = len(unsupported_blocks) + total_blocks -= unsupportedBlocksChildren["estimatedTotalHits"] + total_components -= unsupportedBlocksChildren["estimatedTotalHits"] + unsupported_percentage = (unsupported_blocks_count / total_blocks) * 100 + + state = "success" + if unsupported_blocks_count: + state = "partial" + + # Checks if this migration reaches the block limit + content_library = ContentLibrary.objects.get_by_key(target_key) + assert content_library.learning_package_id is not None + target_item_counts = get_all_drafts(content_library.learning_package_id).count() + if (target_item_counts + total_blocks - unsupported_blocks_count) > blocks_limit: + state = "block_limit_reached" + + return { + "state": state, + "unsupported_blocks": unsupported_blocks_count, + "unsupported_percentage": unsupported_percentage, + "blocks_limit": blocks_limit, + "total_blocks": total_blocks, + "total_components": total_components, + "sections": sections, + "subsections": subsections, + "units": units, + } diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py index 643d94d2250c..43a0b03ee3c9 100644 --- a/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py +++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py @@ -289,3 +289,18 @@ class BlockMigrationInfoSerializer(serializers.Serializer): source_key = serializers.CharField() target_key = serializers.CharField(allow_null=True) unsupported_reason = serializers.CharField(allow_null=True) + + +class PreviewMigrationSerializer(serializers.Serializer): + """ + Serializer for the preview migration response. + """ + state = serializers.CharField() + unsupported_blocks = serializers.IntegerField() + unsupported_percentage = serializers.FloatField() + blocks_limit = serializers.IntegerField() + total_blocks = serializers.IntegerField() + total_components = serializers.IntegerField() + sections = serializers.IntegerField() + subsections = serializers.IntegerField() + units = serializers.IntegerField() diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py index 208825e0c01a..d154d2424763 100644 --- a/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py +++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py @@ -10,6 +10,7 @@ LibraryCourseMigrationViewSet, MigrationInfoViewSet, MigrationViewSet, + PreviewMigration, ) ROUTER = SimpleRouter() @@ -25,4 +26,5 @@ path('', include(ROUTER.urls)), path('migration_info/', MigrationInfoViewSet.as_view(), name='migration-info'), path('migration_blocks/', BlockMigrationInfo.as_view(), name='migration-blocks'), + path('migration_preview/', PreviewMigration.as_view(), name='migration-preview'), ] diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py index d55adb6a53e2..fa46a2c57005 100644 --- a/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py +++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/views.py @@ -41,6 +41,7 @@ MigrationInfoResponseSerializer, ModulestoreMigrationSerializer, StatusWithModulestoreMigrationsSerializer, + PreviewMigrationSerializer, ) log = logging.getLogger(__name__) @@ -668,3 +669,88 @@ def get(self, request: Request): ] serializer = BlockMigrationInfoSerializer(data, many=True) return Response(serializer.data) + + +class PreviewMigration(APIView): + """ + Retrieve the summary preview of the migration given a source key and a target key + + It returns the migration block information for each block migrated by a specific task. + + API Endpoints + ------------- + GET /api/modulestore_migrator/v1/migration_preview/ + Retrieve the summary preview of the migration given a source key and a target key + + Query parameters: + source_key (str): Source content key + Example: ?source_key=course-v1:UNIX+UX1+2025_T3 + target_key (str): target content key + Example: ?target_key=lib:UNIX:CIT1 + + Example request: + GET /api/modulestore_migrator/v1/migration_blocks/?source_key=course_key&target_key=library_key + + Example response: + """ + + permission_classes = (IsAuthenticated,) + authentication_classes = ( + BearerAuthenticationAllowInactiveUser, + JwtAuthentication, + SessionAuthenticationAllowInactiveUser, + ) + + @apidocs.schema( + parameters=[ + apidocs.string_parameter( + "target_key", + apidocs.ParameterLocation.QUERY, + description="Target key of the migration", + ), + apidocs.string_parameter( + "source_key", + apidocs.ParameterLocation.QUERY, + description="Source key of the migration", + ), + ], + responses={ + 200: PreviewMigrationSerializer, + 400: "Missing required parameter: target_key/source_key", + 401: "The requester is not authenticated.", + }, + ) + def get(self, request: Request): + """ + Handle the migration info `GET` request + """ + target_key: LibraryLocatorV2 | None + if target_key_param := request.query_params.get("target_key"): + try: + target_key = LibraryLocatorV2.from_string(target_key_param) + except InvalidKeyError: + return Response({"error": f"Bad target_key: {target_key_param}"}, status=400) + else: + return Response({"error": "Target key cannot be blank."}, status=400) + source_key: SourceContextKey | None = None + if source_key_param := request.query_params.get("source_key"): + try: + source_key = CourseLocator.from_string(source_key_param) + except InvalidKeyError: + try: + source_key = LibraryLocator.from_string(source_key_param) + except InvalidKeyError: + return Response({"error": f"Bad source: {source_key_param}"}, status=400) + else: + return Response({"error": "Source key cannot be blank."}, status=400) + + lib_api.require_permission_for_library_key( + target_key, + request.user, + lib_api.permissions.CAN_EDIT_THIS_CONTENT_LIBRARY, + ) + result = migrator_api.preview_migration(source_key, target_key) + + serializer = PreviewMigrationSerializer(result) + + return Response(serializer.data) diff --git a/cms/djangoapps/modulestore_migrator/tests/test_api.py b/cms/djangoapps/modulestore_migrator/tests/test_api.py index c9e2fc3b587b..937729a8b844 100644 --- a/cms/djangoapps/modulestore_migrator/tests/test_api.py +++ b/cms/djangoapps/modulestore_migrator/tests/test_api.py @@ -3,7 +3,8 @@ """ import pytest -from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2 +from unittest.mock import patch +from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2, CourseLocator from openedx_learning.api import authoring as authoring_api from organizations.tests.factories import OrganizationFactory @@ -29,7 +30,11 @@ def setUp(self): self.user = UserFactory(password=self.user_password, is_staff=True) self.organization = OrganizationFactory(name="My Org", short_name="myorg") self.lib_key_v1 = LibraryLocator.from_string("library-v1:myorg+old") + self.lib_key_v1_2 = LibraryLocator.from_string("library-v1:myorg+old2") + self.lib_key_v1_3 = LibraryLocator.from_string("library-v1:myorg+old3") LibraryFactory.create(org="myorg", library="old", display_name="Old Library", modulestore=self.store) + LibraryFactory.create(org="myorg", library="old2", display_name="Old Library 2", modulestore=self.store) + LibraryFactory.create(org="myorg", library="old3", display_name="Old Library 3", modulestore=self.store) self.lib_key_v2_1 = LibraryLocatorV2.from_string("lib:myorg:1") self.lib_key_v2_2 = LibraryLocatorV2.from_string("lib:myorg:2") lib_api.create_library(org=self.organization, slug="1", title="Test Library 1") @@ -58,6 +63,90 @@ def setUp(self): ] # We load this last so that it has an updated list of children. self.lib_v1 = self.store.get_library(self.lib_key_v1) + self.course_key = CourseLocator.from_string('course-v1:TestOrg+TestCourse+TestRun') + + # Create containers and blocks for legacy libraries + # Old Library 2 + for c in ["X", "Y", "Z"]: + BlockFactory.create( + display_name=f"Unit {c}", + category="vertical", + location=self.lib_key_v1_2.make_usage_key("vertical", c), + parent_location=self.lib_key_v1_2.make_usage_key("library", "library"), + user_id=self.user.id, publish_item=False, + ) + for c in ["X", "Y"]: + BlockFactory.create( + display_name=f"Subsection {c}", + category="sequential", + location=self.lib_key_v1_2.make_usage_key("sequential", c), + parent_location=self.lib_key_v1_2.make_usage_key("library", "library"), + user_id=self.user.id, publish_item=False, + ) + BlockFactory.create( + display_name="Section X", + category="chapter", + location=self.lib_key_v1_2.make_usage_key("chapter", "X"), + parent_location=self.lib_key_v1_2.make_usage_key("library", "library"), + user_id=self.user.id, publish_item=False, + ) + for c in ["X", "Y", "Z"]: + BlockFactory.create( + display_name=f"HTML {c}", + category="html", + location=self.lib_key_v1_2.make_usage_key("html", c), + parent_location=self.lib_key_v1_2.make_usage_key("vertical", c), + user_id=self.user.id, publish_item=False, + ) + + # Old Library 3 + for c in ["X", "Y", "Z"]: + BlockFactory.create( + display_name=f"Unit {c}", + category="vertical", + location=self.lib_key_v1_3.make_usage_key("vertical", c), + parent_location=self.lib_key_v1_3.make_usage_key("library", "library"), + user_id=self.user.id, publish_item=False, + ) + for c in ["X", "Y"]: + BlockFactory.create( + display_name=f"Subsection {c}", + category="sequential", + location=self.lib_key_v1_3.make_usage_key("sequential", c), + parent_location=self.lib_key_v1_3.make_usage_key("library", "library"), + user_id=self.user.id, publish_item=False, + ) + BlockFactory.create( + display_name="Section X", + category="chapter", + location=self.lib_key_v1_3.make_usage_key("chapter", "X"), + parent_location=self.lib_key_v1_3.make_usage_key("library", "library"), + user_id=self.user.id, publish_item=False, + ) + for c in ["X", "Y", "Z"]: + BlockFactory.create( + display_name=f"Html {c}", + category="html", + location=self.lib_key_v1_3.make_usage_key("html", c), + parent_location=self.lib_key_v1_3.make_usage_key("vertical", c), + user_id=self.user.id, publish_item=False, + ) + for c in ["A"]: + BlockFactory.create( + display_name=f"Item Bank {c}", + category="item_bank", + location=self.lib_key_v1_3.make_usage_key("item_bank", c), + parent_location=self.lib_key_v1_3.make_usage_key("vertical", "X"), + user_id=self.user.id, publish_item=False, + ) + for c in ["B"]: + BlockFactory.create( + display_name=f"Invalid {c}", + category="invalid", + location=self.lib_key_v1_3.make_usage_key("invalid", c), + parent_location=self.lib_key_v1_3.make_usage_key("vertical", "X"), + user_id=self.user.id, publish_item=False, + ) def test_start_migration_to_library(self): """ @@ -566,3 +655,296 @@ def test_migration_api_for_various_scenarios(self): forwarded_blocks = api.get_forwarding_for_blocks(all_source_usage_keys) assert forwarded_blocks[self.source_html_keys[1]].target_key.context_key == self.lib_key_v2_1 assert forwarded_blocks[self.source_unit_keys[1]].target_key.context_key == self.lib_key_v2_1 + + def _get_summary_from_migration(self, migration, expected_state, blocks_limit): + """ + Manually calculate the summary from the migration data + """ + blocks = api.get_migration_blocks(migration.pk) + + summary = { + "state": expected_state, + "unsupported_blocks": 0, + "unsupported_percentage": 0, + "blocks_limit": blocks_limit, + "total_blocks": 0, + "total_components": 0, + "sections": 0, + "subsections": 0, + "units": 0, + } + + for key, block in blocks.items(): + block_type = key.block_type + print(block_type) + summary['total_blocks'] += 1 + if block_type not in ['vertical', 'sequential', 'chapter']: + summary['total_components'] += 1 + if block.is_failed: + summary['unsupported_blocks'] += 1 + elif block_type == 'vertical': + summary['units'] += 1 + elif block_type == 'sequential': + summary['subsections'] += 1 + elif block_type == 'chapter': + summary['sections'] += 1 + + if summary['unsupported_blocks']: + summary['unsupported_percentage'] = summary['unsupported_blocks'] * 100 / summary['total_blocks'] + + return summary + + @patch('cms.djangoapps.modulestore_migrator.api.read_api.fetch_block_types') + @patch('cms.djangoapps.modulestore_migrator.api.read_api.get_all_blocks_from_context') + def test_preview_migration_success(self, mock_get_blocks, mock_fetch_block_types): + """ + Test the preview migration summary in the success state + + This tests compare the summary generated by `preview_migration` with the + data generated by a migration. + """ + user = UserFactory() + api.start_migration_to_library( + user=user, + source_key=self.lib_key_v1_2, + target_library_key=self.lib_key_v2_1, + target_collection_slug=None, + composition_level=CompositionLevel.Section, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, + preserve_url_slugs=True, + forward_source_to_target=False, + ) + migration = list(api.get_migrations(self.lib_key_v1_2))[0] + summary = self._get_summary_from_migration(migration, "success", 1000) + + mock_get_blocks.return_value = [ + { + 'block_type': 'html', + 'block_id': '16', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@16', + }, + { + 'block_type': 'html', + 'block_id': '17', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@17', + }, + { + 'block_type': 'html', + 'block_id': '18', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@18', + }, + { + 'block_type': 'chapter', + 'block_id': '19', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@chapter+block@19', + }, + { + 'block_type': 'sequential', + 'block_id': '20', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@20', + }, + { + 'block_type': 'sequential', + 'block_id': '21', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@21', + }, + { + 'block_type': 'vertical', + 'block_id': '22', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@22', + }, + { + 'block_type': 'vertical', + 'block_id': '23', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@23', + }, + { + 'block_type': 'vertical', + 'block_id': '23', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@2', + }, + ] + mock_fetch_block_types.return_value = { + "estimatedTotalHits": 0, + } + + with self.settings(MAX_BLOCKS_PER_CONTENT_LIBRARY=1000): + results = api.preview_migration(self.course_key, self.lib_key_v2_1) + assert results == summary + + @patch('cms.djangoapps.modulestore_migrator.api.read_api.fetch_block_types') + @patch('cms.djangoapps.modulestore_migrator.api.read_api.get_all_blocks_from_context') + def test_preview_migration_partial(self, mock_get_blocks, mock_fetch_block_types): + """ + Test the preview migration summary in the partial state + + This tests compare the summary generated by `preview_migration` with the + data generated by a migration. + """ + user = UserFactory() + api.start_migration_to_library( + user=user, + source_key=self.lib_key_v1_3, + target_library_key=self.lib_key_v2_1, + target_collection_slug=None, + composition_level=CompositionLevel.Section, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, + preserve_url_slugs=True, + forward_source_to_target=False, + ) + migration = list(api.get_migrations(self.lib_key_v1_3))[0] + summary = self._get_summary_from_migration(migration, "partial", 1000) + + mock_get_blocks.return_value = [ + { + 'block_type': 'html', + 'block_id': '16', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@16', + }, + { + 'block_type': 'html', + 'block_id': '17', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@17', + }, + { + 'block_type': 'html', + 'block_id': '18', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@18', + }, + { + 'block_type': 'chapter', + 'block_id': '19', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@chapter+block@19', + }, + { + 'block_type': 'sequential', + 'block_id': '20', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@20', + }, + { + 'block_type': 'sequential', + 'block_id': '21', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@21', + }, + { + 'block_type': 'vertical', + 'block_id': '22', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@22', + }, + { + 'block_type': 'vertical', + 'block_id': '23', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@23', + }, + { + 'block_type': 'vertical', + 'block_id': '23', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@2', + }, + { + 'block_type': 'invalid', + 'block_id': '24', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@invalid+block@24', + }, # Invalid + { + 'block_type': 'item_bank', + 'block_id': '25', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@item_bank+block@25', + }, # Invalid + { + 'block_type': 'html', + 'block_id': '26', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@26', + }, # Child of item bank + { + 'block_type': 'html', + 'block_id': '27', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@27', + }, # Child of item bank + ] + mock_fetch_block_types.return_value = { + "estimatedTotalHits": 2, + } + + # The unsupported children are not included in the summary + with self.settings(MAX_BLOCKS_PER_CONTENT_LIBRARY=1000): + results = api.preview_migration(self.course_key, self.lib_key_v2_1) + assert results == summary + + @patch('cms.djangoapps.modulestore_migrator.api.read_api.fetch_block_types') + @patch('cms.djangoapps.modulestore_migrator.api.read_api.get_all_blocks_from_context') + def test_preview_migration_block_limit(self, mock_get_blocks, mock_fetch_block_types): + """ + Test the preview migration summary in the block_limit_reached state + + This tests compare the summary generated by `preview_migration` with the + data generated by a migration. + """ + user = UserFactory() + api.start_migration_to_library( + user=user, + source_key=self.lib_key_v1_2, + target_library_key=self.lib_key_v2_1, + target_collection_slug=None, + composition_level=CompositionLevel.Section, + repeat_handling_strategy=RepeatHandlingStrategy.Skip, + preserve_url_slugs=True, + forward_source_to_target=False, + ) + migration = list(api.get_migrations(self.lib_key_v1_2))[0] + summary = self._get_summary_from_migration(migration, "block_limit_reached", 10) + + mock_get_blocks.return_value = [ + { + 'block_type': 'html', + 'block_id': '16', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@16', + }, + { + 'block_type': 'html', + 'block_id': '17', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@17', + }, + { + 'block_type': 'html', + 'block_id': '18', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@18', + }, + { + 'block_type': 'chapter', + 'block_id': '19', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@chapter+block@19', + }, + { + 'block_type': 'sequential', + 'block_id': '20', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@20', + }, + { + 'block_type': 'sequential', + 'block_id': '21', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@21', + }, + { + 'block_type': 'vertical', + 'block_id': '22', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@22', + }, + { + 'block_type': 'vertical', + 'block_id': '23', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@23', + }, + { + 'block_type': 'vertical', + 'block_id': '23', + 'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@2', + }, + ] + mock_fetch_block_types.return_value = { + "estimatedTotalHits": 0, + } + + with self.settings(MAX_BLOCKS_PER_CONTENT_LIBRARY=10): + results = api.preview_migration(self.course_key, self.lib_key_v2_1) + assert results == summary diff --git a/cms/djangoapps/modulestore_migrator/tests/test_rest_api.py b/cms/djangoapps/modulestore_migrator/tests/test_rest_api.py index 83c214015ed9..78472d41098c 100644 --- a/cms/djangoapps/modulestore_migrator/tests/test_rest_api.py +++ b/cms/djangoapps/modulestore_migrator/tests/test_rest_api.py @@ -32,6 +32,7 @@ BulkMigrationViewSet, MigrationInfoViewSet, MigrationViewSet, + PreviewMigration, ) from openedx.core.djangoapps.content_libraries import api as lib_api @@ -1109,3 +1110,147 @@ def test_get_block_migration_info_without_library_access(self, mock_lib_api): response = self.view(request) assert response.status_code == status.HTTP_403_FORBIDDEN + + +class TestPreviewMigration(TestCase): + """ + Test the PreviewMigration.get() endpoint. + """ + def setUp(self): + """Set up test fixtures.""" + self.factory = APIRequestFactory() + self.view = PreviewMigration.as_view() + + self.user = User.objects.create_user( + username='testuser', + email='testuser@test.com', + password='password' + ) + + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.migrator_api') + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api') + def test_preview_migration_success(self, mock_lib_api, mock_migrator_api): + """ + Test successful retrieval of preview migration. + """ + mock_lib_api.require_permission_for_library_key.return_value = None + + expected = { + "state": "partial", + "unsupported_blocks": 4, + "unsupported_percentage": 25, + "blocks_limit": 1000, + "total_blocks": 20, + "total_components": 10, + "sections": 2, + "subsections": 3, + "units": 5, + } + + mock_migrator_api.preview_migration.return_value = expected + + request = self.factory.get( + '/api/modulestore_migrator/v1/migration_preview/', + { + 'target_key': 'lib:TestOrg:TestLibrary', + 'source_key': 'course-v1:TestOrg+TestCourse+TestRun', + } + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_200_OK + + for key, value in expected.items(): + assert response.data[key] == value + + @patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api') + def test_preview_migration_without_library_access(self, mock_lib_api): + """ + Test that users without library view access get 403 Forbidden. + """ + mock_lib_api.require_permission_for_library_key.side_effect = PermissionDenied( + "User lacks permission to view this library" + ) + + request = self.factory.get( + '/api/modulestore_migrator/v1/migration_preview/', + { + 'target_key': 'lib:TestOrg:TestLibrary', + 'source_key': 'course-v1:TestOrg+TestCourse+TestRun', + } + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_preview_migration_missing_target_key(self): + """ + Test that missing target_key parameter returns 400 Bad Request. + """ + request = self.factory.get( + '/api/modulestore_migrator/v1/migration_preview/', + { + 'source_key': 'course-v1:TestOrg+TestCourse+TestRun', + } + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert 'target' in response.data.get('error', '').lower() + + def test_preview_migration_invalid_target_key(self): + """ + Test that invalid target_key returns 400 Bad Request. + """ + request = self.factory.get( + '/api/modulestore_migrator/v1/migration_preview/', + { + 'target_key': 'not-a-valid-key', + 'source_key': 'course-v1:TestOrg+TestCourse+TestRun', + } + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_preview_migration_missing_source_key(self): + """ + Test that missing target_key parameter returns 400 Bad Request. + """ + request = self.factory.get( + '/api/modulestore_migrator/v1/migration_preview/', + { + 'target_key': 'lib:TestOrg:TestLibrary', + } + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert 'source' in response.data.get('error', '').lower() + + def test_preview_migration_invalid_source_key(self): + """ + Test that invalid target_key returns 400 Bad Request. + """ + request = self.factory.get( + '/api/modulestore_migrator/v1/migration_preview/', + { + 'target_key': 'lib:TestOrg:TestLibrary', + 'source_key': 'not-a-valid-key', + } + ) + force_authenticate(request, user=self.user) + + response = self.view(request) + + assert response.status_code == status.HTTP_400_BAD_REQUEST diff --git a/openedx/core/djangoapps/content/search/api.py b/openedx/core/djangoapps/content/search/api.py index 1771eaa37f88..9cdb51941996 100644 --- a/openedx/core/djangoapps/content/search/api.py +++ b/openedx/core/djangoapps/content/search/api.py @@ -5,10 +5,11 @@ import logging import time +from collections.abc import Iterator from contextlib import contextmanager, nullcontext from datetime import datetime, timedelta, timezone from functools import wraps -from typing import Callable, Generator +from typing import Callable, Generator, cast from django.conf import settings from django.contrib.auth import get_user_model @@ -61,6 +62,8 @@ STUDIO_INDEX_SUFFIX = "studio_content" +Filter = str | list[str | list[str]] + if hasattr(settings, "MEILISEARCH_INDEX_PREFIX"): STUDIO_INDEX_NAME = settings.MEILISEARCH_INDEX_PREFIX + STUDIO_INDEX_SUFFIX else: @@ -981,3 +984,96 @@ def generate_user_token_for_studio_search(request): "index_name": STUDIO_INDEX_NAME, "api_key": restricted_api_key, } + + +def force_array(extra_filter: Filter | None = None) -> list[str]: + """ + Convert a filter value into a list of strings. + + Strings are wrapped in a list, lists are returned as-is (cast to `list[str]`), + and None results in an empty list. + """ + if isinstance(extra_filter, str): + return [extra_filter] + if isinstance(extra_filter, list): + return cast(list[str], extra_filter) + return [] + + +def fetch_block_types(extra_filter: Filter | None = None): + """ + Fetch the block types facet distribution for the search results. + + This data may not always be 100% accurate / up to date because it's based + on the search index, so this should only be used for analysis/estimation + purposes. + + Params: + - extra_filter: Filters the query. Example: ['context_key = "course-v1:SampleTaxonomyOrg1+CC22+CC22"'] + + Return example: + { + ... + 'estimatedTotalHits': 5, + 'facetDistribution': { + 'block_type': { + 'html': 2, + 'problem': 1, + 'video': 2, + } + }, + } + """ + extra_filter_formatted = force_array(extra_filter) + + client = _get_meilisearch_client() + index = client.get_index(STUDIO_INDEX_NAME) + + response = index.search( + "", + { + "facets": ["block_type"], + "filter": extra_filter_formatted, + "limit": 0, + } + ) + + return response + + +def get_all_blocks_from_context( + context_key: str, + extra_attributes_to_retrieve: list[str] | None = None, +) -> Iterator[dict]: + """ + Lazily yields all blocks for a given context key using Meilisearch pagination. + Meilisearch works with limits of 1000 maximum; ensuring we obtain all blocks + requires making several queries. + + This data may not always be 100% accurate / up to date because it's based + on the search index, so this should only be used for analysis/estimation + purposes. + """ + limit = 1000 + offset = 0 + + client = _get_meilisearch_client() + index = client.get_index(STUDIO_INDEX_NAME) + + while True: + response = index.search( + "", + { + "filter": [f'context_key = "{context_key}"'], + "limit": limit, + "offset": offset, + "attributesToRetrieve": ["usage_key"] + (extra_attributes_to_retrieve or []), + } + ) + + yield from response["hits"] + + if response["estimatedTotalHits"] <= offset + limit: + break + + offset += limit diff --git a/openedx/core/djangoapps/content/search/tests/test_api.py b/openedx/core/djangoapps/content/search/tests/test_api.py index e1b6f8fe16b0..f0213eae75a2 100644 --- a/openedx/core/djangoapps/content/search/tests/test_api.py +++ b/openedx/core/djangoapps/content/search/tests/test_api.py @@ -1233,3 +1233,68 @@ def test_section_in_usbsections(self, mock_meilisearch) -> None: ], any_order=True, ) + + @override_settings(MEILISEARCH_ENABLED=True) + def test_fetch_block_types(self, mock_meilisearch): + from openedx.core.djangoapps.content.search.api import fetch_block_types + + mock_index = mock_meilisearch.return_value.get_index.return_value + fetch_block_types('context_key = test') + + mock_index.search.assert_called_once_with( + "", + { + "facets": ["block_type"], + "filter": ['context_key = test'], + "limit": 0, + } + ) + + @override_settings(MEILISEARCH_ENABLED=True) + def test_get_all_blocks_from_context(self, mock_meilisearch): + from openedx.core.djangoapps.content.search.api import get_all_blocks_from_context + + mock_index = mock_meilisearch.return_value.get_index.return_value + expected_result = [ + {"usage_key": "block-v1:test+type@html+block@1"}, + {"usage_key": "block-v1:test+type@video+block@2"}, + ] + + # Simulate two pages: one with results and one empty (while loop ends) + mock_index.search.side_effect = [ + { + "hits": expected_result, + "estimatedTotalHits": 1200, + }, + { + "hits": [], + "estimatedTotalHits": 1200, + }, + ] + + result = list(get_all_blocks_from_context( + context_key="course-v1:TestOrg+TestCourse+TestRun", + extra_attributes_to_retrieve=["display_name"], + )) + + assert result == expected_result + assert mock_index.search.call_count == 2 + mock_index.search.assert_any_call( + "", + { + "filter": ['context_key = "course-v1:TestOrg+TestCourse+TestRun"'], + "limit": 1000, + "offset": 0, + "attributesToRetrieve": ["usage_key", "display_name"], + } + ) + + mock_index.search.assert_any_call( + "", + { + "filter": ['context_key = "course-v1:TestOrg+TestCourse+TestRun"'], + "limit": 1000, + "offset": 1000, + "attributesToRetrieve": ["usage_key", "display_name"], + } + )