fix: index and entity link sync issues on parent block deletion (#37541)

Meilisearch index documents were not synced properly when any block with children blocks like units, subsections, sections etc. were being deleted as the `XBLOCK_DELETED` is only triggered for the deleted block.
This PR fixes it by deleting all index documents that contain the deleted block in its `breadcrumbs` field as only blocks that are children of this block will have it its breadcrumbs field.

Similarly, the entity links that store links between course and library blocks was not synced properly due to children `ContainerLinks` not being deleted.

Author: navinkarkera

cms/djangoapps/contentstore/signals/handlers.py

@@ -221,6 +221,7 @@ def handle_item_deleted(**kwargs):
             id_list.add(block.location)
 
         ComponentLink.objects.filter(downstream_usage_key__in=id_list).delete()
+        ContainerLink.objects.filter(downstream_usage_key__in=id_list).delete()
 
 
 @receiver(GRADING_POLICY_CHANGED)

openedx/core/djangoapps/content/search/api.py

@@ -638,7 +638,7 @@ def add_with_children(block):
     _update_index_docs(docs)
 
 
-def delete_index_doc(key: OpaqueKey) -> None:
+def delete_index_doc(key: OpaqueKey, *, delete_children: bool = False) -> None:
     """
     Deletes the document for the given XBlock from the search index
 
@@ -647,6 +647,37 @@ def delete_index_doc(key: OpaqueKey) -> None:
     """
     doc = searchable_doc_for_key(key)
     _delete_index_doc(doc[Fields.id])
+    if delete_children:
+        _delete_documents(f'{Fields.breadcrumbs}.{Fields.usage_key} = "{key}"')
+
+
+def delete_docs_with_context_key(key: OpaqueKey) -> None:
+    """
+    Delete all docs for given context key
+    """
+    _delete_documents(f'{Fields.context_key} = "{key}"')
+
+
+def _delete_documents(filter_query: str) -> None:
+    """
+    Deletes all documents from the search index that match the given filter
+
+    Args:
+        filter (str): The query to use when filtering documents
+    """
+    if not filter_query:
+        return
+
+    client = _get_meilisearch_client()
+    current_rebuild_index_name = _get_running_rebuild_index_name()
+
+    tasks = []
+    if current_rebuild_index_name:
+        # If there is a rebuild in progress, the document will also be removed from the new index.
+        tasks.append(client.index(current_rebuild_index_name).delete_documents(filter=filter_query))
+    tasks.append(client.index(STUDIO_INDEX_NAME).delete_documents(filter=filter_query))
+
+    _wait_for_meili_tasks(tasks)
 
 
 def _delete_index_doc(doc_id) -> None:

openedx/core/djangoapps/content/search/documents.py

@@ -48,8 +48,9 @@ class Fields:
     org = "org"
     access_id = "access_id"  # .models.SearchAccess.id
     # breadcrumbs: an array of {"display_name": "..."} entries. First one is the name of the course/library itself.
-    # After that is the name of any parent Section/Subsection/Unit/etc.
+    # After that is the name of any parent Section/Subsection/Unit/etc and its usage_key.
     # It's a list of dictionaries because for now we just include the name of each but in future we may add their IDs.
+    # Example: [{"display_name": "My course"}, {"display_name": "Section1", "usage_key": "..."}]}
     breadcrumbs = "breadcrumbs"
     # tags (dictionary)
     # See https://blog.meilisearch.com/nested-hierarchical-facets-guide/

openedx/core/djangoapps/content/search/handlers.py

@@ -43,6 +43,7 @@
 from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
 from openedx.core.djangoapps.content.search.models import SearchAccess
 from openedx.core.djangoapps.content_libraries import api as lib_api
+from xmodule.modulestore.django import SignalHandler
 
 from .api import (
     only_if_meilisearch_enabled,
@@ -51,6 +52,7 @@
     upsert_item_containers_index_docs,
 )
 from .tasks import (
+    delete_course_index_docs,
     delete_library_block_index_doc,
     delete_library_container_index_doc,
     delete_xblock_index_doc,
@@ -344,3 +346,12 @@ def handle_reindex_on_signal(**kwargs):
         return
 
     upsert_course_blocks_docs.delay(str(course_data.course_key))
+
+
+@receiver(SignalHandler.course_deleted)
+def listen_for_course_delete(sender, course_key, **kwargs):  # pylint: disable=unused-argument
+    """
+    Catches the signal that a course has been deleted
+    and removes its entry from the Course About Search index.
+    """
+    delete_course_index_docs.delay(str(course_key))

openedx/core/djangoapps/content/search/index_config.py

@@ -26,6 +26,7 @@
     Fields.last_published,
     Fields.content + "." + Fields.problem_types,
     Fields.publish_status,
+    Fields.breadcrumbs + "." + Fields.usage_key,
 ]
 
 # Mark which attributes are used for keyword search, in order of importance:

openedx/core/djangoapps/content/search/tasks.py

@@ -59,7 +59,8 @@ def delete_xblock_index_doc(usage_key_str: str) -> None:
 
     log.info("Updating content index document for XBlock with id: %s", usage_key)
 
-    api.delete_index_doc(usage_key)
+    # Delete children index data for course blocks.
+    api.delete_index_doc(usage_key, delete_children=True)
 
 
 @shared_task(base=LoggedTask, autoretry_for=(MeilisearchError, ConnectionError))
@@ -168,3 +169,17 @@ def delete_library_container_index_doc(container_key_str: str) -> None:
     log.info("Deleting content index document for library block with id: %s", container_key)
 
     api.delete_index_doc(container_key)
+
+
+@shared_task(base=LoggedTask, autoretry_for=(MeilisearchError, ConnectionError))
+@set_code_owner_attribute
+def delete_course_index_docs(course_key_str: str) -> None:
+    """
+    Celery task to delete the content index documents for a Course
+    """
+    course_key = CourseKey.from_string(course_key_str)
+
+    log.info("Deleting all index documents related to course_key: %s", course_key)
+
+    # Delete children index data for course blocks.
+    api.delete_docs_with_context_key(course_key)

openedx/core/djangoapps/content/search/tests/test_api.py

@@ -618,14 +618,18 @@ def test_index_xblock_tags(self, mock_meilisearch) -> None:
     @override_settings(MEILISEARCH_ENABLED=True)
     def test_delete_index_xblock(self, mock_meilisearch) -> None:
         """
-        Test deleting an XBlock doc from the index.
+        Test deleting an XBlock doc and its children docs from the index.
         """
-        api.delete_index_doc(self.sequential.usage_key)
+        api.delete_index_doc(self.sequential.usage_key, delete_children=True)
 
         mock_meilisearch.return_value.index.return_value.delete_document.assert_called_once_with(
             self.doc_sequential['id']
         )
 
+        mock_meilisearch.return_value.index.return_value.delete_documents.assert_called_once_with(
+            filter=f'breadcrumbs.usage_key = "{self.sequential.usage_key}"'
+        )
+
     @override_settings(MEILISEARCH_ENABLED=True)
     def test_index_library_block_metadata(self, mock_meilisearch) -> None:
         """
@@ -821,6 +825,17 @@ def test_delete_index_library_block(self, mock_meilisearch) -> None:
             self.doc_problem1['id']
         )
 
+    @override_settings(MEILISEARCH_ENABLED=True)
+    def test_delete_docs_with_context_key(self, mock_meilisearch) -> None:
+        """
+        Test deleting a all Block docs from the index using context_key.
+        """
+        api.delete_docs_with_context_key(self.course.id)
+
+        mock_meilisearch.return_value.index.return_value.delete_documents.assert_called_once_with(
+            filter=f'context_key = "{self.course.id}"'
+        )
+
     @override_settings(MEILISEARCH_ENABLED=True)
     def test_index_content_library_metadata(self, mock_meilisearch) -> None:
         """