452 fix issue with validation with schemas with external references (#894)

* rename documentation utility

* fix error with schemas with external references in validation mode

* apply formatting changes

---------

Co-authored-by: Logende <[email protected]>

Author: Logende

meta_configurator/src/data/managedValidation.ts

@@ -8,6 +8,7 @@ import {sizeOf} from '@/utility/sizeOf';
 
 // Import worker as ESM
 import ValidationWorker from '@/workers/validationWorker?worker';
+import {removeExternalReferences} from '@/schema/removeExternalReferences.ts';
 
 export class ManagedValidation {
   private worker: Worker;
@@ -70,14 +71,17 @@ export class ManagedValidation {
       this.isValidationOngoing = true;
 
       const data = getDataForMode(this.mode).data.value;
-      const schema = getSchemaForMode(this.mode).schemaRaw.value;
 
       if (sizeOf(data) > useSettings().value.performance.maxDocumentSizeForValidation) {
         this.currentValidationResult.value = new ValidationResult([]);
         this.isValidationOngoing = false;
         return;
       }
 
+      // duplicate the schema and remove external references to avoid ajv trying to fetch them and throwing an error if they are not available
+      const schema = JSON.parse(JSON.stringify(getSchemaForMode(this.mode).schemaRaw.value));
+      removeExternalReferences(schema);
+
       try {
         this.currentValidationResult.value = await this.validateWithWorker(data, schema);
       } catch (err) {

meta_configurator/src/schema/removeExternalReferences.ts

@@ -0,0 +1,32 @@
+import {isExternalRef} from '@/schema/externalReferences.ts';
+
+export function removeExternalReferences(schema: any): number {
+  let externalRefsRemoved = 0;
+
+  // recursively go over while schema and remove all external refs
+  if (typeof schema === 'object' && schema !== null) {
+    if (
+      schema.$ref !== undefined &&
+      typeof schema.$ref === 'string' &&
+      isExternalRef(schema.$ref)
+    ) {
+      // remove external ref
+      delete schema.$ref;
+      externalRefsRemoved++;
+    }
+    // recursively check all properties of the schema
+    for (const key in schema) {
+      if (schema.hasOwnProperty(key)) {
+        externalRefsRemoved += removeExternalReferences(schema[key]);
+      }
+    }
+  }
+  // handle arrays
+  if (Array.isArray(schema)) {
+    for (const item of schema) {
+      externalRefsRemoved += removeExternalReferences(item);
+    }
+  }
+
+  return externalRefsRemoved;
+}

…igurator/src/schema/resolveReferences.ts …ema/resolveReferencesForDocumentation.tsmeta_configurator/src/schema/resolveReferences.ts renamed to meta_configurator/src/schema/resolveReferencesForDocumentation.ts meta_configurator/src/schema/resolveReferences.ts renamed to meta_configurator/src/schema/resolveReferencesForDocumentation.ts

@@ -8,7 +8,7 @@ import {doesSchemaHaveType} from '@/schema/schemaReadingUtils';
 import {useErrorService} from '@/utility/errorServiceInstance';
 
 // this code is not recursive and also mainly developed for the documentation view, not sure how generally applicable it is
-export function resolveReferences(subSchema: any, rootSchema: TopLevelSchema): any {
+export function resolveReferencesForDocumentation(subSchema: any, rootSchema: TopLevelSchema): any {
   if (!subSchema) {
     return new Set();
   }
@@ -19,7 +19,7 @@ export function resolveReferences(subSchema: any, rootSchema: TopLevelSchema): a
     // copy subSchema
     const subSchemaWithoutRef = cloneDeep(subSchema);
     delete subSchemaWithoutRef.$ref;
-    const resolvedSchema = resolveReferences(schemaAtRef, rootSchema);
+    const resolvedSchema = resolveReferencesForDocumentation(schemaAtRef, rootSchema);
     try {
       return mergeAllOfs([subSchemaWithoutRef, resolvedSchema]);
     } catch (error) {

meta_configurator/src/schema/validationService.ts

@@ -10,6 +10,7 @@ import {
   getTopLevelSchemaId,
   ValidationResult,
 } from '@/schema/validationUtils';
+import {removeExternalReferences} from '@/schema/removeExternalReferences.ts';
 
 /**
  * Service for validating data against a JSON schema.
@@ -32,10 +33,13 @@ export class ValidationService {
   }
 
   private initValidationFunction() {
-    this._ajv = getMatchingAjvVersion(this.topLevelSchema);
+    // duplicate the schema and remove external references to avoid ajv trying to fetch them and throwing an error if they are not available
+    let schema = JSON.parse(JSON.stringify(this.topLevelSchema));
+    removeExternalReferences(schema);
+    this._ajv = getMatchingAjvVersion(schema);
     addFormats(this._ajv);
-    const topLevelSchemaId = getTopLevelSchemaId(this.topLevelSchema);
-    this._ajv.addSchema(this.topLevelSchema, topLevelSchemaId);
+    const topLevelSchemaId = getTopLevelSchemaId(schema);
+    this._ajv.addSchema(schema, topLevelSchemaId);
     this._validationFunction = this._ajv.getSchema(topLevelSchemaId);
   }
 

meta_configurator/src/utility/documentation/documentationUtils.ts

@@ -1,5 +1,9 @@
 import type {JsonSchemaObjectType, TopLevelSchema} from '@/schema/jsonSchemaType';
-import {collectReferences, findTargetPath, resolveReferences} from '@/schema/resolveReferences';
+import {
+  collectReferences,
+  findTargetPath,
+  resolveReferencesForDocumentation,
+} from '@/schema/resolveReferencesForDocumentation.ts';
 import type {Path} from '@/utility/path';
 import {pathToAscii} from '@/utility/pathUtils';
 import {formatRegistry} from '@/dataformats/formatRegistry';
@@ -96,7 +100,10 @@ export function generateSchemaInstance(
   visitedReferences = new Set([...visitedReferences, ...collectReferences(schema, rootSchema)]);
 
   // resolve the references of the current schema if needed
-  const resolvedSchema: JsonSchemaObjectType = resolveReferences(schema, rootSchema);
+  const resolvedSchema: JsonSchemaObjectType = resolveReferencesForDocumentation(
+    schema,
+    rootSchema
+  );
 
   const type = resolvedSchema?.type ?? 'any';
   if (type === 'string') return '{string}';

meta_configurator/src/utility/documentation/schemaToMarkdown.ts

@@ -11,7 +11,7 @@ import {
   SchemaNodeData,
 } from '@/schema/graph-representation/schemaGraphTypes';
 import type {JsonSchemaObjectType, TopLevelSchema} from '@/schema/jsonSchemaType';
-import {resolveReferences} from '@/schema/resolveReferences';
+import {resolveReferencesForDocumentation} from '@/schema/resolveReferencesForDocumentation.ts';
 import {hasOutgoingEdge} from '@/schema/graph-representation/graphUtils';
 import {
   escapeMarkdown,
@@ -267,7 +267,10 @@ function writeObjectNode(
     md.push(`</details>`);
   }
 
-  const instance = generateSchemaInstance(resolveReferences(node.schema, rootSchema), rootSchema);
+  const instance = generateSchemaInstance(
+    resolveReferencesForDocumentation(node.schema, rootSchema),
+    rootSchema
+  );
   if (instance) {
     md.push('#### Example\n');
     md.push('```' + useSettings().value.dataFormat);