implement different performance improvements for large files (#733)

* implement different performance improvements for large files

* apply formatting changes

---------

Co-authored-by: Logende <[email protected]>

Author: Logende

meta_configurator/src/components/panels/code-editor/AceEditor.vue

@@ -3,7 +3,7 @@
  Synchronized with file data from the store.
  -->
 <script setup lang="ts">
-import {onMounted, watch} from 'vue';
+import {computed, onMounted, type Ref, ref, watch} from 'vue';
 import type {Editor} from 'brace';
 import * as ace from 'brace';
 import 'brace/mode/javascript';
@@ -21,6 +21,7 @@ import {useSettings} from '@/settings/useSettings';
 import {modeToDocumentTypeDescription, SessionMode} from '@/store/sessionMode';
 import {setupAceMode, setupAceProperties} from '@/components/panels/shared-components/aceUtils';
 import Message from 'primevue/message';
+import {sizeOf} from '@/utility/sizeOf';
 
 const props = defineProps<{
   sessionMode: SessionMode;
@@ -30,27 +31,29 @@ const settings = useSettings();
 
 // random id is used to enable multiple Ace Editors of same sessionMode on the same page
 const editor_id = 'code-editor-' + props.sessionMode + '-' + Math.random();
+let editor: Ref<Editor | undefined> = ref(undefined);
 
 onMounted(() => {
-  const editor: Editor = ace.edit(editor_id);
-  setupAceMode(editor, settings.value);
-  setupAceProperties(editor, settings.value);
+  editor.value = ace.edit(editor_id);
+  setupAceMode(editor.value, settings.value);
+  setupAceProperties(editor.value, settings.value);
 
-  setupLinkToData(editor, props.sessionMode);
-  setupLinkToCurrentSelection(editor, props.sessionMode);
-  setupAnnotationsFromValidationErrors(editor, props.sessionMode);
+  setupLinkToData(editor.value, props.sessionMode);
+  setupLinkToCurrentSelection(editor.value, props.sessionMode);
+  setupAnnotationsFromValidationErrors(editor.value, props.sessionMode);
 
   if (isEditorReadOnly()) {
-    editor.setReadOnly(true);
+    editor.value.setReadOnly(true);
   }
 });
 
 // watch for changes in the data format and update the editor accordingly
 watch(
   () => settings.value.dataFormat,
   _ => {
-    const editor: Editor = ace.edit(editor_id);
-    editor.setReadOnly(isEditorReadOnly());
+    if (editor.value) {
+      editor.value.setReadOnly(isEditorReadOnly());
+    }
   }
 );
 
@@ -63,6 +66,18 @@ function isEditorReadOnly(): boolean {
     (mode === SessionMode.SchemaEditor || mode === SessionMode.Settings) && dataFormat === 'xml'
   );
 }
+
+const featuresDisabledForPerformance = computed(() => {
+  if (!editor.value) {
+    return false;
+  }
+  const performanceSettings = settings.value.performance;
+  const editorContentSize = sizeOf(editor.value.getValue());
+  return (
+    editorContentSize > performanceSettings.maxDocumentSizeForValidation ||
+    editorContentSize > performanceSettings.maxDocumentSizeForCursorSynchronization
+  );
+});
 </script>
 
 <template>
@@ -71,6 +86,10 @@ function isEditorReadOnly(): boolean {
     underlying JSON {{ modeToDocumentTypeDescription(props.sessionMode) }} document, because of
     ambiguity and technical restrictions in XML to JSON conversion.</Message
   >
+  <Message v-if="featuresDisabledForPerformance" severity="warn"
+    >Some editor features are disabled for performance reasons due to the large size of the
+    document.</Message
+  >
   <div class="h-full" :id="editor_id" />
 </template>
 

meta_configurator/src/components/panels/code-editor/setupLinkToSelectionAndData.ts

@@ -10,6 +10,8 @@ import {
 import type {SessionMode} from '@/store/sessionMode';
 import {getDataForMode, getSessionForMode} from '@/data/useDataLink';
 import {watchImmediate} from '@vueuse/core/index';
+import {useSettings} from '@/settings/useSettings';
+import {sizeOf} from '@/utility/sizeOf';
 
 // variables to prevent updating functions to trigger each other
 let selectionChangeFromOutside = false;
@@ -42,6 +44,14 @@ function setupCursorPositionToSelectedPath(editor: Editor, mode: SessionMode) {
         // do not attempt to determine the path when the text does not have valid syntax
         return;
       }
+      if (
+        sizeOf(editor.getValue()) >
+        useSettings().value.performance.maxDocumentSizeForCursorSynchronization
+      ) {
+        // do not determine the path when the document is exceeding the maximum size
+        return;
+      }
+
       try {
         const newPath = determinePath(editor);
         const session = getSessionForMode(mode);
@@ -70,6 +80,14 @@ function setupSelectedPathToCursorPosition(editor: Editor, mode: SessionMode) {
         return;
       }
       selectionChangeFromOutside = true;
+
+      if (
+        sizeof(editor.getValue()) >
+        useSettings().value.performance.maxDocumentSizeForCursorSynchronization
+      ) {
+        // do not determine and update the cursor position when the document is exceeding the maximum size
+        return;
+      }
       updateCursorPositionBasedOnPath(editor, newSelectedElement);
     }
   );

meta_configurator/src/components/panels/gui-editor/configTreeNodeResolver.ts

@@ -195,6 +195,11 @@ export class ConfigTreeNodeResolver {
         this.createObjectChildrenTreeNodes(mode, {absolutePath, relativePath, schema, depth})
       );
     }
+
+    if (children.length > settings.value.performance.maximumShownChildrenInGuiEditor) {
+      children = children.slice(0, settings.value.performance.maximumShownChildrenInGuiEditor);
+    }
+
     return children;
   }
 
@@ -456,7 +461,12 @@ export class ConfigTreeNodeResolver {
         );
       });
     }
-    if (this.shouldAddAddItemNode(schema, data)) {
+    let exceedsChildrenLimit = false;
+    if (children.length > settings.value.performance.maximumShownChildrenInGuiEditor) {
+      children = children.slice(0, settings.value.performance.maximumShownChildrenInGuiEditor);
+      exceedsChildrenLimit = true;
+    }
+    if (this.shouldAddAddItemNode(schema, data) && !exceedsChildrenLimit) {
       return children.concat(
         this.createAddItemTreeNode({absolutePath, relativePath, schema, depth: depth + 1}, children)
       );

meta_configurator/src/data/managedValidation.ts

@@ -1,7 +1,9 @@
 import {computed} from 'vue';
 import {getDataForMode, getSchemaForMode} from '@/data/useDataLink';
 import {SessionMode} from '@/store/sessionMode';
-import {ValidationService} from '@/schema/validationService';
+import {ValidationResult, ValidationService} from '@/schema/validationService';
+import {useSettings} from '@/settings/useSettings';
+import {sizeOf} from '@/utility/sizeOf';
 
 export class ManagedValidation {
   constructor(public mode: SessionMode) {}
@@ -12,6 +14,10 @@ export class ManagedValidation {
   });
 
   public currentValidationResult = computed(() => {
+    const data = getDataForMode(this.mode).data.value;
+    if (sizeOf(data) > useSettings().value.performance.maxDocumentSizeForValidation) {
+      return new ValidationResult([]);
+    }
     return this.currentValidationService.value.validate(getDataForMode(this.mode).data.value);
   });
 }

meta_configurator/src/settings/defaultSettingsData.ts

@@ -7,6 +7,11 @@ export const SETTINGS_DATA_DEFAULT = {
   toolbarTitle: 'MetaConfigurator',
   hideSchemaEditor: false,
   hideSettings: false,
+  performance: {
+    maxDocumentSizeForValidation: 1000000, // 1 MB
+    maxDocumentSizeForCursorSynchronization: 1000000, // 1 MB
+    maximumShownChildrenInGuiEditor: 50,
+  },
   codeEditor: {
     fontSize: 14,
     tabSize: 2,

meta_configurator/src/settings/settingsSchema.ts

@@ -36,6 +36,35 @@ export const SETTINGS_SCHEMA: TopLevelSchema = {
       description: 'If set to true, the complete settings view will be hidden.',
       default: false,
     },
+    performance: {
+      type: 'object',
+      required: ['maxDocumentSizeForValidation', 'maxDocumentSizeForCursorSynchronization'],
+      additionalProperties: false,
+      description: 'Performance related settings belong here.',
+      properties: {
+        maxDocumentSizeForValidation: {
+          type: 'integer',
+          description:
+            'The maximum size of the document to validate in bytes. If the document is larger, it will not be validated.',
+          default: 1000000, // 1 MB
+          minimum: 1000,
+        },
+        maxDocumentSizeForCursorSynchronization: {
+          type: 'integer',
+          description:
+            'The maximum size of the document to synchronize the cursor position in bytes. If the document is larger, the cursor position will not be synchronized.',
+          default: 1000000, // 1 MB
+          minimum: 1000,
+        },
+        maximumShownChildrenInGuiEditor: {
+          type: 'integer',
+          description:
+            'The maximum amount of child nodes to be shown in the GUI editor per parent node. If the document has more children than this value, those will not be shown in the GUI editor, but still exist in the document and can be edited by other panels.',
+          default: 50,
+          minimum: 5,
+        },
+      },
+    },
     codeEditor: {
       type: 'object',
       required: ['fontSize'],

meta_configurator/src/settings/settingsTypes.ts

@@ -6,6 +6,7 @@ export interface SettingsInterfaceRoot {
   toolbarTitle: string;
   hideSchemaEditor: boolean;
   hideSettings: boolean;
+  performance: SettingsInterfacePerformance;
   codeEditor: SettingsInterfaceCodeEditor;
   guiEditor: SettingsInterfaceGuiEditor;
   schemaDiagram: SettingsInterfaceSchemaDiagram;
@@ -17,6 +18,12 @@ export interface SettingsInterfaceRoot {
   aiIntegration: SettingsInterfaceAiIntegraton;
 }
 
+export interface SettingsInterfacePerformance {
+  maxDocumentSizeForValidation: number; // in bytes
+  maxDocumentSizeForCursorSynchronization: number; // in bytes
+  maximumShownChildrenInGuiEditor: number;
+}
+
 export interface SettingsInterfaceCodeEditor {
   fontSize: number;
   tabSize: number;

meta_configurator/src/utility/sizeOf.ts

@@ -0,0 +1,35 @@
+// returns a size estimation
+// source: https://stackoverflow.com/a/11900218/4743756
+export function sizeOf(object: any) {
+  const objectList: any[] = [];
+  const stack = [object];
+  let bytes = 0;
+
+  while (stack.length) {
+    const value = stack.pop();
+
+    switch (typeof value) {
+      case 'boolean':
+        bytes += 4;
+        break;
+      case 'string':
+        bytes += value.length * 2;
+        break;
+      case 'number':
+        bytes += 8;
+        break;
+      case 'object':
+        if (!objectList.includes(value)) {
+          objectList.push(value);
+          for (const prop in value) {
+            if (value.hasOwnProperty(prop)) {
+              stack.push(value[prop]);
+            }
+          }
+        }
+        break;
+    }
+  }
+
+  return bytes;
+}