diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ff435232f..36ef716b1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -2,9 +2,9 @@ name: E2E Tests on: push: - branches: [main] + branches: [develop] pull_request: - branches: [main] + branches: [develop] jobs: e2e: diff --git a/.github/workflows/github-pages.yml b/.github/workflows/github-pages.yml index f5ebbad57..f94d6cdfe 100644 --- a/.github/workflows/github-pages.yml +++ b/.github/workflows/github-pages.yml @@ -18,6 +18,7 @@ jobs: env: USE_META_CONFIGURATOR_BASE_PATH: true # Set to true for GitHub Pages deployment VITE_FRONTEND_HOSTNAME: https://metaconfigurator.github.io/meta-configurator + EXPERIMENTAL: true run: | cd meta_configurator npm ci diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml new file mode 100644 index 000000000..4e5db4d6f --- /dev/null +++ b/.github/workflows/tag-version.yml @@ -0,0 +1,28 @@ +name: Tag Version +on: + push: + branches: + - main + - develop +permissions: + contents: write +jobs: + tag-version: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Create tag if version is new + run: | + VERSION=$(node -p "require('./meta_configurator/package.json').version") + TAG="v$VERSION" + if git rev-parse "$TAG" >/dev/null 2>&1; then + echo "Tag $TAG already exists, skipping." + else + git tag "$TAG" + git push origin "$TAG" + echo "Created and pushed tag $TAG." + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a1588b71..e36f903fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- +## [2.3.0] - 2026-04-14 + +### Changed + +- Remove prototypical STML mapping in favor of more powerful Jsonata +- Refactor the code base to use a newly implemented JSON Schema visitor pattern instead of having schema traversal logic implemented in different places + +### Fixed + +- Fix schema diagram: it now correctly draws multiple edges if a sub-schema defines its own structure and additionally has a reference + +## [2.2.0] - 2026-03-27 + +### Added + +- Add word-wrap for text editor +- Add `experimental` tag in the About page for the experimental deployment +- Add workflow to automatically generate a git tag when a PR is merged into `main` or `develop` with an incremented version in the package.json + +### Changed + +- Update multiple dependencies (picomatch, handlebars, yaml) + ## [2.1.0] - 2026-03-25 Initial versioned release. Introduces semantic versioning, a structured branching model (`develop` / `main`), and contribution guidelines. diff --git a/README.md b/README.md index 4345e6b22..354f454b8 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,9 @@ pages = {1--9} MetaConfigurator runs entirely **inside your browser** - it does **not** send your schemas, data, or anything you type to any server. The website itself is just a static page delivered by GitHub Pages (like downloading a PDF or image), and all the work happens locally on your computer. -The only exception is if you **click the “Share Snapshot” button**. Then, and only then, the snapshot you create is sent to a **University of Stuttgart** server so you can share a unique link with others. +The exception is if you **click the “Share Snapshot” button**. Then, and only then, the snapshot you create is sent to a **University of Stuttgart** server so you can share a unique link with others. + +Additionally, if you use the optional AI assistance functionality, relevant subsets of your schema and data are shared with the AI endpoint you configure in the MetaConfigurator settings (by default OpenAI). This will not accidentally happen, as an authentification key from the AI endpoint provider is needed to use the AI assistance features. **See our [full Privacy Policy](PRIVACY.md)** for more information. diff --git a/meta_configurator/env.d.ts b/meta_configurator/env.d.ts index dbb4c627d..55e938483 100644 --- a/meta_configurator/env.d.ts +++ b/meta_configurator/env.d.ts @@ -1,3 +1,4 @@ /// declare const __APP_VERSION__: string; +declare const __APP_EXPERIMENTAL__: boolean; diff --git a/meta_configurator/package-lock.json b/meta_configurator/package-lock.json index bff15f637..e8bd8b8fb 100644 --- a/meta_configurator/package-lock.json +++ b/meta_configurator/package-lock.json @@ -26,7 +26,7 @@ "dagrejs": "^0.2.1", "fast-xml-parser": "^5.0.8", "flowbite": "^1.6.5", - "handlebars": "^4.7.8", + "handlebars": "^4.7.9", "js-yaml": "^4.1.0", "json-cst": "^1.2.0", "json-pointer": "^0.6.2", @@ -46,7 +46,7 @@ "vue": "^3.5.13", "vue-flow": "^0.3.0", "vue-router": "^4.3.2", - "yaml": "^2.3.1" + "yaml": "^2.8.3" }, "devDependencies": { "@ls-lint/ls-lint": "^2.2.3", @@ -1475,10 +1475,11 @@ } }, "node_modules/@rollup/pluginutils/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, + "license": "MIT", "engines": { "node": ">=12" }, @@ -5727,9 +5728,9 @@ } }, "node_modules/handlebars": { - "version": "4.7.8", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", - "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "version": "4.7.9", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", + "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", "license": "MIT", "dependencies": { "minimist": "^1.2.5", @@ -8556,10 +8557,11 @@ "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, + "license": "MIT", "engines": { "node": ">=8.6" }, @@ -9086,10 +9088,11 @@ } }, "node_modules/pretty-quick/node_modules/picomatch": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-3.0.1.tgz", - "integrity": "sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-3.0.2.tgz", + "integrity": "sha512-cfDHL6LStTEKlNilboNtobT/kEa30PtAf2Q1OgszfrG/rpVl1xaFWT9ktfkS306GmHgmnad1Sw4wabhlvFtsTw==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -10815,9 +10818,9 @@ } }, "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "license": "MIT", "engines": { "node": ">=12" @@ -12034,14 +12037,18 @@ "dev": true }, "node_modules/yaml": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.0.tgz", - "integrity": "sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "license": "ISC", "bin": { "yaml": "bin.mjs" }, "engines": { - "node": ">= 14" + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" } }, "node_modules/yargs": { diff --git a/meta_configurator/package.json b/meta_configurator/package.json index ba1c7fec9..726b0488d 100644 --- a/meta_configurator/package.json +++ b/meta_configurator/package.json @@ -1,6 +1,6 @@ { "name": "meta-configurator", - "version": "2.1.0", + "version": "2.3.0", "private": true, "scripts": { "dev": "vite", @@ -42,7 +42,7 @@ "dagrejs": "^0.2.1", "fast-xml-parser": "^5.0.8", "flowbite": "^1.6.5", - "handlebars": "^4.7.8", + "handlebars": "^4.7.9", "js-yaml": "^4.1.0", "json-cst": "^1.2.0", "json-pointer": "^0.6.2", @@ -62,7 +62,7 @@ "vue": "^3.5.13", "vue-flow": "^0.3.0", "vue-router": "^4.3.2", - "yaml": "^2.3.1" + "yaml": "^2.8.3" }, "devDependencies": { "@ls-lint/ls-lint": "^2.2.3", diff --git a/meta_configurator/src/components/panels/ai-prompts/ApiKey.vue b/meta_configurator/src/components/panels/ai-prompts/ApiKey.vue index 80ac6f0bd..c251f78dd 100644 --- a/meta_configurator/src/components/panels/ai-prompts/ApiKey.vue +++ b/meta_configurator/src/components/panels/ai-prompts/ApiKey.vue @@ -5,16 +5,14 @@ Component for displaying the OpenAI API key input. import {type Ref, ref} from 'vue'; import Password from 'primevue/password'; import SelectButton from 'primevue/selectbutton'; -import {getApiKeyRef, getIsPersistKeyRef} from '@/utility/ai/apiKey'; - -const isShowPersistOption = false; // currently the option of whether to persist the key is not shown because without persistence the key currently can not be accessed +import {getApiKeyRef, getRememberInTabRef} from '@/utility/ai/apiKey'; const apiKey: Ref = getApiKeyRef(); -const isPersistKey: Ref = getIsPersistKeyRef(); +const rememberInTab: Ref = getRememberInTabRef(); -const persistOptions = ref([ - {name: 'true', value: true}, - {name: 'false', value: false}, +const rememberOptions = ref([ + {name: 'Remember in this tab', value: true}, + {name: 'Forget on refresh', value: false}, ]); @@ -28,17 +26,18 @@ const persistOptions = ref([ possible without permanently connecting your credit card with your account. Check this link for pricing.
-
MetaConfigurator by default uses the gpt-4o-mini model, which has very low cost. For improved results you can change to more performant models in the settings tab. +
+
+ Your key is stored only in your browser and sent directly to your chosen provider. It is never + sent to MetaConfigurator servers. Key: - - Persist: + diff --git a/meta_configurator/src/components/panels/ai-prompts/aiPromptUtils.ts b/meta_configurator/src/components/panels/ai-prompts/aiPromptUtils.ts index 1a2d12724..cf4b59cf1 100644 --- a/meta_configurator/src/components/panels/ai-prompts/aiPromptUtils.ts +++ b/meta_configurator/src/components/panels/ai-prompts/aiPromptUtils.ts @@ -38,6 +38,4 @@ function hasMoreOpeningBrackets(input: string): boolean { return openingCount > closingCount; } -export function getApiKey(): string { - return localStorage.getItem('openai_api_key') || ''; -} +export {getApiKey} from '@/utility/ai/apiKey'; diff --git a/meta_configurator/src/components/panels/code-editor/AceEditor.vue b/meta_configurator/src/components/panels/code-editor/AceEditor.vue index c76a93203..f4ca7b33e 100644 --- a/meta_configurator/src/components/panels/code-editor/AceEditor.vue +++ b/meta_configurator/src/components/panels/code-editor/AceEditor.vue @@ -35,6 +35,11 @@ let editor: Ref = ref(undefined); onMounted(() => { editor.value = ace.edit(editor_id); + + editor.value.getSession().setUseWrapMode(true); + editor.value.setOption('wrap', true); + editor.value.setOption('hScrollBarAlwaysVisible', false); + setupAceMode(editor.value, settings.value); setupAceProperties(editor.value, settings.value); @@ -45,6 +50,13 @@ onMounted(() => { if (isEditorReadOnly()) { editor.value.setReadOnly(true); } + + // watch for changes in the editor container size and resize the editor accordingly + const observer = new ResizeObserver(() => { + editor.value?.resize(); + }); + const el = document.getElementById(editor_id); + if (el) observer.observe(el); }); // watch for changes in the data format and update the editor accordingly diff --git a/meta_configurator/src/components/toolbar/dialogs/AboutDialog.vue b/meta_configurator/src/components/toolbar/dialogs/AboutDialog.vue index 592959c36..ebd6ec2b2 100644 --- a/meta_configurator/src/components/toolbar/dialogs/AboutDialog.vue +++ b/meta_configurator/src/components/toolbar/dialogs/AboutDialog.vue @@ -2,8 +2,9 @@ Dialog that shows information about the application and the licenses of the used icons. Emits an update:visible event when the dialog is closed. --> - - + diff --git a/meta_configurator/src/components/toolbar/dialogs/data-mapping/DataMappingDialog.vue b/meta_configurator/src/components/toolbar/dialogs/data-mapping/DataMappingDialog.vue index b81915c3d..0953c8d04 100644 --- a/meta_configurator/src/components/toolbar/dialogs/data-mapping/DataMappingDialog.vue +++ b/meta_configurator/src/components/toolbar/dialogs/data-mapping/DataMappingDialog.vue @@ -9,7 +9,6 @@ import Message from 'primevue/message'; import ApiKey from '@/components/panels/ai-prompts/ApiKey.vue'; import {SessionMode} from '@/store/sessionMode'; import {getDataForMode} from '@/data/useDataLink'; -import {DataMappingServiceStml} from '@/data-mapping/stml/dataMappingServiceStml'; import {DataMappingServiceJsonata} from '@/data-mapping/jsonata/dataMappingServiceJsonata'; import type {DataMappingService} from '@/data-mapping/dataMappingService'; import type {Editor} from 'brace'; @@ -35,19 +34,15 @@ const isLoadingMapping = ref(false); const settings = useSettings(); -const mappingServiceTypes = ['Advanced (JSONata)', 'SimpleTransformationMappingLanguage (STML)']; +const mappingServiceTypes = ['Advanced (JSONata)']; const mappingServiceWarnings = [ 'The JSONata mapping service is very expressive flexible, but may generate invalid mappings for complex inputs, which have to manually be corrected.', - 'The STML mapping service usually generates valid mappings, but it can perform only simple source to target path mappings and value transformations. WARNING: It supports executing arbitrary JavaScript functions as transformations, which may lead to security issues if the input is not properly sanitized.', ]; const selectedMappingServiceType: Ref = ref(mappingServiceTypes[0]); const mappingService: Ref = computed(() => { - if (selectedMappingServiceType.value === 'SimpleTransformationMappingLanguage (STML)') { - return new DataMappingServiceStml(); - } if (selectedMappingServiceType.value === 'Advanced (JSONata)') { return new DataMappingServiceJsonata(); } diff --git a/meta_configurator/src/data-mapping/jsonata/dataMappingServiceJsonata.ts b/meta_configurator/src/data-mapping/jsonata/dataMappingServiceJsonata.ts index 3c60d28d5..c10fea486 100644 --- a/meta_configurator/src/data-mapping/jsonata/dataMappingServiceJsonata.ts +++ b/meta_configurator/src/data-mapping/jsonata/dataMappingServiceJsonata.ts @@ -22,13 +22,6 @@ export class DataMappingServiceJsonata implements DataMappingService { userComments: string ): Promise<{config: string; success: boolean; message: string}> { const inputDataSubset = trimDataToMaxSize(input); - console.log( - 'Reduced input data from ' + - JSON.stringify(input).length / 1024 + - ' KB to ' + - JSON.stringify(inputDataSubset).length / 1024 + - ' KB' - ); // infer schema for input data const inputFileSchema = inferJsonSchema(inputDataSubset); @@ -43,25 +36,6 @@ export class DataMappingServiceJsonata implements DataMappingService { const inputFileSchemaStr = JSON.stringify(inputFileSchema); const targetSchemaStr = JSON.stringify(targetSchema); const inputDataSubsetStr = JSON.stringify(inputDataSubset); - console.log( - 'Sizes of the different input files in KB:' + - ' jsonata example files: ' + - ( - (jsonataReferenceStr.length + - jsonataInputExampleStr.length + - jsonataInputExampleSchemaStr.length + - jsonataExpressionStr.length + - jsonataOutputExampleStr.length + - jsonataOutputExampleSchemaStr.length) / - 1024 - ).toFixed(2) + - ' inputFileSchema: ' + - (inputFileSchemaStr.length / 1024).toFixed(2) + - ' targetSchema: ' + - (targetSchemaStr.length / 1024).toFixed(2) + - ' inputDataSubset: ' + - (inputDataSubsetStr.length / 1024).toFixed(2) - ); const resultPromise = queryJsonataExpression( apiKey, jsonataReferenceStr, @@ -124,12 +98,25 @@ export class DataMappingServiceJsonata implements DataMappingService { return result; } - removeSpecialCharactersRecursive(data: any) { - // TODO + removeSpecialCharactersRecursive(data: any): void { + if (Array.isArray(data)) { + data.forEach(item => this.removeSpecialCharactersRecursive(item)); + } else if (data !== null && typeof data === 'object') { + for (const key of Object.keys(data)) { + const sanitizedKey = key.replace(/[^a-zA-Z0-9_]/g, '_'); + if (sanitizedKey !== key) { + data[sanitizedKey] = data[key]; + delete data[key]; + } + this.removeSpecialCharactersRecursive(data[sanitizedKey]); + } + } } - sanitizeMappingConfig(config: string, input: any): string { - return config; // TODO + sanitizeMappingConfig(config: string, _input: any): string { + // JSONata natively supports special characters in property names via backtick syntax, + // so no transformation of the mapping config expression is required. + return config; } validateMappingConfig(config: string, input: any): {success: boolean; message: string} { diff --git a/meta_configurator/src/data-mapping/stml/__tests__/applyTransformations.test.ts b/meta_configurator/src/data-mapping/stml/__tests__/applyTransformations.test.ts deleted file mode 100644 index 43e234a93..000000000 --- a/meta_configurator/src/data-mapping/stml/__tests__/applyTransformations.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import {describe, expect, it, vi} from 'vitest'; -import {applyTransformations} from '../applyTransformations'; -import {type Transformation} from '../dataMappingTypes'; - -// avoid constructing useDataLink store through imports, it is not required for this component -vi.mock('@/data/useDataLink', () => ({ - getSchemaForMode: vi.fn(), - getDataForMode: vi.fn(), - useCurrentData: vi.fn(), - useCurrentSchema: vi.fn(), - getUserSelectionForMode: vi.fn(), - getValidationForMode: vi.fn(), - getSessionForMode: vi.fn(), -})); - -describe('test transformations for data mapping', () => { - const inputData = { - people: [ - { - name: 'John Doe', - age: 30, - address: { - street: '123 Main St', - city: 'Anytown', - zip: '12345', - }, - hobbies: ['reading', 'gaming'], - }, - { - name: 'Jane Smith', - age: 25, - address: { - street: '456 Elm St', - city: 'Othertown', - zip: '67890', - }, - hobbies: ['cooking', 'traveling'], - }, - ], - books: [ - { - title: 'Book 1', - }, - { - title: 'Book 2', - }, - ], - year: 2050, - }; - - const transformations: Transformation[] = [ - { - operationType: 'function', - sourcePath: '/people/%INDEX_A%/age', - function: 'x + 1', - }, - { - operationType: 'function', - sourcePath: '/people/%INDEX_A%/name', - function: 'x.toUpperCase()', - }, - { - operationType: 'valueMapping', - sourcePath: '/books/%INDEX_A%/title', - valueMapping: { - 'Book 1': 'First Book', - 'Book 2': 'Second Book', - }, - }, - ]; - - it('test transformation using the mathFormula operator', () => { - const outputData = applyTransformations(inputData, transformations); - expect(outputData.people[0].age).toEqual(31); - expect(outputData.people[1].age).toEqual(26); - }); - - it('test transformation using the stringOperation operator', () => { - const outputData = applyTransformations(inputData, transformations); - expect(outputData.people[0].name).toEqual('JOHN DOE'); - expect(outputData.people[1].name).toEqual('JANE SMITH'); - }); - - it('test transformation using the valueMapping operator', () => { - const outputData = applyTransformations(inputData, transformations); - expect(outputData.books[0].title).toEqual('First Book'); - expect(outputData.books[1].title).toEqual('Second Book'); - }); -}); diff --git a/meta_configurator/src/data-mapping/stml/__tests__/dataMappingUtils.test.ts b/meta_configurator/src/data-mapping/stml/__tests__/dataMappingUtils.test.ts deleted file mode 100644 index 8b0609cbe..000000000 --- a/meta_configurator/src/data-mapping/stml/__tests__/dataMappingUtils.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import {describe, expect, it, vi} from 'vitest'; -import {normalizeJsonPointer, pathToNormalizedJsonPointer} from '../dataMappingUtilsStml'; - -// avoid constructing useDataLink store through imports, it is not required for this component -vi.mock('@/data/useDataLink', () => ({ - getSchemaForMode: vi.fn(), - getDataForMode: vi.fn(), - useCurrentData: vi.fn(), - useCurrentSchema: vi.fn(), - getUserSelectionForMode: vi.fn(), - getValidationForMode: vi.fn(), - getSessionForMode: vi.fn(), -})); - -describe('test data mapping utils', () => { - it('test pathToNormalizedJsonPointer', () => { - const path1 = ['people', 0, 'name']; - const expectedPointer1 = '/people/%INDEX_A%/name'; - const result1 = pathToNormalizedJsonPointer(path1, true); - expect(result1).toEqual(expectedPointer1); - - const path2 = ['people', 10, 'address', 'street']; - const expectedPointer2 = '/people/%INDEX_A%/address/street'; - const result2 = pathToNormalizedJsonPointer(path2, true); - expect(result2).toEqual(expectedPointer2); - - // example with multiple indices - const path3 = ['people', 5, 'hobbies', 4]; - const expectedPointer3 = '/people/%INDEX_A%/hobbies/%INDEX_B%'; - const result3 = pathToNormalizedJsonPointer(path3, true); - expect(result3).toEqual(expectedPointer3); - - // example where the replaceIndexByPlaceholder option is set false - const path4 = ['people', 5, 'hobbies', 4]; - const expectedPointer4 = '/people/5/hobbies/4'; - const result4 = pathToNormalizedJsonPointer(path4, false); - expect(result4).toEqual(expectedPointer4); - }); - - it('test normalizeJsonPointer', () => { - const pointer1 = '/people/0/name'; - const expectedNormPointer1 = '/people/%INDEX_A%/name'; - const result1 = normalizeJsonPointer(pointer1, true); - expect(result1).toEqual(expectedNormPointer1); - - const pointer2 = '/people/3/address/street'; - const expectedNormPointer2 = '/people/%INDEX_A%/address/street'; - const result2 = normalizeJsonPointer(pointer2, true); - expect(result2).toEqual(expectedNormPointer2); - - // example with multiple indices - const pointer3 = '/people/4/hobbies/0'; - const expectedNormPointer3 = '/people/%INDEX_A%/hobbies/%INDEX_B%'; - const result3 = normalizeJsonPointer(pointer3, true); - expect(result3).toEqual(expectedNormPointer3); - - // example where the replaceIndexByPlaceholder option is set false - const pointer4 = '/people/5/hobbies/4'; - const expectedNormPointer4 = '/people/5/hobbies/4'; - const result4 = normalizeJsonPointer(pointer4, false); - expect(result4).toEqual(expectedNormPointer4); - - // example where the pointer starts with a hashtag - const pointer5 = '#/people/0/name'; - const expectedNormPointer5 = '/people/%INDEX_A%/name'; - const result5 = normalizeJsonPointer(pointer5, true); - expect(result5).toEqual(expectedNormPointer5); - - // example where the pointer starts without a slash - const pointer6 = 'people/0/name'; - const expectedNormPointer6 = '/people/%INDEX_A%/name'; - const result6 = normalizeJsonPointer(pointer6, true); - expect(result6).toEqual(expectedNormPointer6); - }); -}); diff --git a/meta_configurator/src/data-mapping/stml/__tests__/extractPathsFromDocument.test.ts b/meta_configurator/src/data-mapping/stml/__tests__/extractPathsFromDocument.test.ts deleted file mode 100644 index 3fd8da31e..000000000 --- a/meta_configurator/src/data-mapping/stml/__tests__/extractPathsFromDocument.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -import {describe, expect, it, vi} from 'vitest'; -import { - extractInvalidSourcePathsFromConfig, - extractSourcePaths, - extractSuitableSourcePaths, -} from '../extractPathsFromDocument'; -import type {DataMappingConfig} from '../dataMappingTypes'; - -// avoid constructing useDataLink store through imports, it is not required for this component -vi.mock('@/data/useDataLink', () => ({ - getSchemaForMode: vi.fn(), - getDataForMode: vi.fn(), - useCurrentData: vi.fn(), - useCurrentSchema: vi.fn(), - getUserSelectionForMode: vi.fn(), - getValidationForMode: vi.fn(), - getSessionForMode: vi.fn(), -})); - -describe('test extraction of suitable source paths for data mapping', () => { - const inputData = { - people: [ - { - name: 'John Doe', - age: 30, - address: { - street: '123 Main St', - city: 'Anytown', - zip: '12345', - }, - hobbies: ['reading', 'gaming'], - }, - { - name: 'Jane Smith', - age: 25, - address: { - street: '456 Elm St', - city: 'Othertown', - zip: '67890', - }, - hobbies: ['cooking', 'traveling'], - }, - ], - books: [ - { - title: 'Book 1', - }, - { - title: 'Book 2', - }, - ], - year: 2050, - }; - - const mappingConfig: DataMappingConfig = { - // note that only values get copied for which exists a mapping. Map all values except the address - mappings: [ - { - sourcePath: '/people/%INDEX_A%/name', - targetPath: '/person/%INDEX_A%/fullName', - }, - { - sourcePath: '/people/%INDEX_A%/age', - targetPath: '/person/%INDEX_A%/age', - }, - { - sourcePath: '/books/%INDEX_A%/title', - targetPath: '/library/%INDEX_A%/bookTitle', - }, - // note that the hobby mappings do not correspond to the input data, as they are objects and not just simple strings - { - sourcePath: '/people/%INDEX_A%/hobbies/%INDEX_B%/name', - targetPath: '/person/%INDEX_A%/activities/%INDEX_B%/hobbyName', - }, - { - sourcePath: '/people/%INDEX_A%/hobbies/%INDEX_B%/type', - targetPath: '/person/%INDEX_A%/activities/%INDEX_B%/hobbyType', - }, - { - sourcePath: '/year', - targetPath: '/year', - }, - ], - transformations: [ - { - operationType: 'mathFormula', - sourcePath: '/people/%INDEX_A%/age', - formula: 'x + 1', - }, - { - operationType: 'stringOperation', - sourcePath: '/people/%INDEX_A%/name', - string: 'uppercase', - }, - { - operationType: 'valueMapping', - sourcePath: '/books/%INDEX_A%/title', - valueMapping: { - 'Book 1': 'First Book', - 'Book 2': 'Second Book', - }, - }, - ], - }; - - it('test extraction of suitable source paths', () => { - const suitableSourcePaths = extractSuitableSourcePaths(inputData); - const expectedSourcePaths = [ - '/people/%INDEX_A%/name', - '/people/%INDEX_A%/age', - '/people/%INDEX_A%/address/street', - '/people/%INDEX_A%/address/city', - '/people/%INDEX_A%/address/zip', - '/people/%INDEX_A%/hobbies/%INDEX_B%', - '/books/%INDEX_A%/title', - '/year', - ]; - expect(suitableSourcePaths).toEqual(expectedSourcePaths); - }); - - it('test extraction of actual source paths from mapping config', () => { - const actualSourcePaths = extractSourcePaths(mappingConfig); - const expectedSourcePaths = [ - '/people/%INDEX_A%/name', - '/people/%INDEX_A%/age', - '/books/%INDEX_A%/title', - '/people/%INDEX_A%/hobbies/%INDEX_B%/name', - '/people/%INDEX_A%/hobbies/%INDEX_B%/type', - '/year', - ]; - expect(actualSourcePaths).toEqual(expectedSourcePaths); - }); - - it('test that invalid source paths in config are properly detected', () => { - const invalidSourcePaths = extractInvalidSourcePathsFromConfig(mappingConfig, inputData); - const expectedInvalidSourcePaths = [ - '/people/%INDEX_A%/hobbies/%INDEX_B%/name', - '/people/%INDEX_A%/hobbies/%INDEX_B%/type', - ]; - expect(invalidSourcePaths).toEqual(expectedInvalidSourcePaths); - }); -}); diff --git a/meta_configurator/src/data-mapping/stml/__tests__/findMatchingPaths.test.ts b/meta_configurator/src/data-mapping/stml/__tests__/findMatchingPaths.test.ts deleted file mode 100644 index 9b1959981..000000000 --- a/meta_configurator/src/data-mapping/stml/__tests__/findMatchingPaths.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -import {describe, expect, it, vi} from 'vitest'; -import {findMatchingPaths} from '../findMatchingPaths'; - -// avoid constructing useDataLink store through imports, it is not required for this component -vi.mock('@/data/useDataLink', () => ({ - getSchemaForMode: vi.fn(), - getDataForMode: vi.fn(), - useCurrentData: vi.fn(), - useCurrentSchema: vi.fn(), - getUserSelectionForMode: vi.fn(), - getValidationForMode: vi.fn(), - getSessionForMode: vi.fn(), -})); - -describe('test path matching', () => { - const inputData = { - people: [ - { - name: 'John Doe', - age: 30, - address: { - street: '123 Main St', - city: 'Anytown', - zip: '12345', - }, - hobbies: ['reading', 'gaming'], - }, - { - name: 'Jane Smith', - age: 25, - address: { - street: '456 Elm St', - city: 'Othertown', - zip: '67890', - }, - hobbies: ['cooking', 'traveling'], - }, - ], - books: [ - { - title: 'Book 1', - }, - { - title: 'Book 2', - }, - ], - year: 2050, - }; - - it('all matching paths should successfully be found, starting with a simple path without an array index', () => { - // test the year path - const sourcePathDef = '/year'; - const expectedPaths = [['year']]; - const result = findMatchingPaths(inputData, sourcePathDef); - expect(result).toEqual(expectedPaths); - }); - - it('all matching paths should successfully be found, handling a single array index', () => { - // test the street path - const sourcePathDef = '/people/%INDEX%/address/street'; - const expectedPaths = [ - ['people', 0, 'address', 'street'], - ['people', 1, 'address', 'street'], - ]; - const result = findMatchingPaths(inputData, sourcePathDef); - expect(result).toEqual(expectedPaths); - - // test the books path - const sourcePathDef2 = '/books/%INDEX%/title'; - const expectedPaths2 = [ - ['books', 0, 'title'], - ['books', 1, 'title'], - ]; - const result2 = findMatchingPaths(inputData, sourcePathDef2); - expect(result2).toEqual(expectedPaths2); - }); - - it('all matching paths should successfully be found, handling nested array indices', () => { - // test the hobbies path - const sourcePathDef = '/people/%INDEX%/hobbies/%INDEX%'; - const expectedPaths = [ - ['people', 0, 'hobbies', 0], - ['people', 0, 'hobbies', 1], - ['people', 1, 'hobbies', 0], - ['people', 1, 'hobbies', 1], - ]; - const result = findMatchingPaths(inputData, sourcePathDef); - expect(result).toEqual(expectedPaths); - }); - - it('the path matching should also support %INDEX_A% to %INDEX_Z% by treating it as %INDEX%', () => { - // test the hobbies path, now using %INDEX_A% and %INDEX_Z% - const sourcePathDef = '/people/%INDEX_A%/hobbies/%INDEX_Z%'; - const expectedPaths = [ - ['people', 0, 'hobbies', 0], - ['people', 0, 'hobbies', 1], - ['people', 1, 'hobbies', 0], - ['people', 1, 'hobbies', 1], - ]; - const result = findMatchingPaths(inputData, sourcePathDef); - expect(result).toEqual(expectedPaths); - }); -}); diff --git a/meta_configurator/src/data-mapping/stml/__tests__/performDataMapping.test.ts b/meta_configurator/src/data-mapping/stml/__tests__/performDataMapping.test.ts deleted file mode 100644 index 52ea29213..000000000 --- a/meta_configurator/src/data-mapping/stml/__tests__/performDataMapping.test.ts +++ /dev/null @@ -1,261 +0,0 @@ -import {describe, expect, it, vi} from 'vitest'; -import {type DataMappingConfig} from '../dataMappingTypes'; -import {performSimpleDataMapping} from '../performDataMapping'; - -// avoid constructing useDataLink store through imports, it is not required for this component -vi.mock('@/data/useDataLink', () => ({ - getSchemaForMode: vi.fn(), - getDataForMode: vi.fn(), - useCurrentData: vi.fn(), - useCurrentSchema: vi.fn(), - getUserSelectionForMode: vi.fn(), - getValidationForMode: vi.fn(), - getSessionForMode: vi.fn(), -})); - -describe('test performing data mappings on a given input file, based on a mapping configuration', () => { - const inputData = { - people: [ - { - name: 'John Doe', - age: 30, - address: { - street: '123 Main St', - city: 'Anytown', - zip: '12345', - }, - hobbies: [ - { - name: 'reading', - type: 'indoor', - }, - { - name: 'gaming', - type: 'indoor', - }, - ], - }, - { - name: 'Jane Smith', - age: 25, - address: { - street: '456 Elm St', - city: 'Othertown', - zip: '67890', - }, - hobbies: [ - { - name: 'cooking', - type: 'indoor', - }, - { - name: 'traveling', - type: 'outdoor', - }, - ], - }, - ], - books: [ - { - title: 'Book 1', - }, - { - title: 'Book 2', - }, - ], - year: 2050, - }; - - const mappingConfig: DataMappingConfig = { - // note that only values get copied for which exists a mapping. Map all values except the address - mappings: [ - { - sourcePath: '/people/%INDEX_A%/name', - targetPath: '/person/%INDEX_A%/fullName', - }, - { - sourcePath: '/people/%INDEX_A%/age', - targetPath: '/person/%INDEX_A%/age', - }, - { - sourcePath: '/books/%INDEX_A%/title', - targetPath: '/library/%INDEX_A%/bookTitle', - }, - { - sourcePath: '/people/%INDEX_A%/hobbies/%INDEX_B%/name', - targetPath: '/person/%INDEX_A%/activities/%INDEX_B%/hobbyName', - }, - { - sourcePath: '/people/%INDEX_A%/hobbies/%INDEX_B%/type', - targetPath: '/person/%INDEX_A%/activities/%INDEX_B%/hobbyType', - }, - { - sourcePath: '/year', - targetPath: '/year', - }, - ], - transformations: [ - { - operationType: 'function', - sourcePath: '/people/%INDEX_A%/age', - function: 'x + 1', - }, - { - operationType: 'function', - sourcePath: '/people/%INDEX_A%/name', - function: 'x.toUpperCase()', - }, - { - operationType: 'valueMapping', - sourcePath: '/books/%INDEX_A%/title', - valueMapping: { - 'Book 1': 'First Book', - 'Book 2': 'Second Book', - }, - }, - ], - }; - - it('test the complete mapping', () => { - const result = performSimpleDataMapping(inputData, mappingConfig); - - expect(result).toEqual({ - person: [ - { - fullName: 'JOHN DOE', - age: 31, - activities: [ - { - hobbyName: 'reading', - hobbyType: 'indoor', - }, - { - hobbyName: 'gaming', - hobbyType: 'indoor', - }, - ], - }, - { - fullName: 'JANE SMITH', - age: 26, - activities: [ - { - hobbyName: 'cooking', - hobbyType: 'indoor', - }, - { - hobbyName: 'traveling', - hobbyType: 'outdoor', - }, - ], - }, - ], - library: [ - { - bookTitle: 'First Book', - }, - { - bookTitle: 'Second Book', - }, - ], - year: 2050, - }); - }); - - it('test the data mapping for a scenario where not all array elements have all expected fields to be mapped', () => { - // create copy of input data and modify it, removing name of first hobby of John and age of Jane - // also remove full book list - const modifiedInputData = JSON.parse(JSON.stringify(inputData)); - delete modifiedInputData.people[0].hobbies[0].name; - delete modifiedInputData.people[1].age; - delete modifiedInputData.books; - - const result = performSimpleDataMapping(modifiedInputData, mappingConfig); - - // expect all data was mapped properly and the missing fields simply ignored - expect(result).toEqual({ - person: [ - { - fullName: 'JOHN DOE', - age: 31, - activities: [ - { - hobbyType: 'indoor', - }, - { - hobbyName: 'gaming', - hobbyType: 'indoor', - }, - ], - }, - { - fullName: 'JANE SMITH', - activities: [ - { - hobbyName: 'cooking', - hobbyType: 'indoor', - }, - { - hobbyName: 'traveling', - hobbyType: 'outdoor', - }, - ], - }, - ], - year: 2050, - }); - }); -}); - -// there used to be a bug that for a target array at root level the array indexes are confused for object keys -describe('test performing data mappings when the root element is an array', () => { - const inputDataArrayRoot = [ - { - name: 'John Doe', - age: 30, - address: { - street: '123 Main St', - city: 'Anytown', - zip: '12345', - }, - }, - { - name: 'Jane Smith', - age: 25, - address: { - street: '456 Elm St', - city: 'Othertown', - zip: '67890', - }, - }, - ]; - - const mappingConfigArrayRoot: DataMappingConfig = { - mappings: [ - { - sourcePath: '/%INDEX_A%/name', - targetPath: '/%INDEX_A%/fullName', - }, - { - sourcePath: '/%INDEX_A%/age', - targetPath: '/%INDEX_A%/age', - }, - ], - transformations: [], - }; - - it('test the complete mapping with array root', () => { - const result = performSimpleDataMapping(inputDataArrayRoot, mappingConfigArrayRoot); - - expect(result).toEqual([ - { - fullName: 'John Doe', - age: 30, - }, - { - fullName: 'Jane Smith', - age: 25, - }, - ]); - }); -}); diff --git a/meta_configurator/src/data-mapping/stml/applyTransformations.ts b/meta_configurator/src/data-mapping/stml/applyTransformations.ts deleted file mode 100644 index f0a10b63c..000000000 --- a/meta_configurator/src/data-mapping/stml/applyTransformations.ts +++ /dev/null @@ -1,57 +0,0 @@ -import {dataAt} from '@/utility/resolveDataAtPath'; -import _ from 'lodash'; -import {findMatchingPaths} from '@/data-mapping/stml/findMatchingPaths'; -import type {Transformation} from '@/data-mapping/stml/dataMappingTypes'; - -export function applyTransformations(inputData: any, transformations: Transformation[]): any { - const outputData = JSON.parse(JSON.stringify(inputData)); // Deep clone input data - - for (const transformation of transformations) { - const sourcePathDef = transformation.sourcePath; - const sourcePathsWhichMatchDef = findMatchingPaths(inputData, sourcePathDef); - - for (const sourcePath of sourcePathsWhichMatchDef) { - const value = dataAt(sourcePath, outputData); - if (value !== undefined) { - const transformedValue = applyTransformationsOnValue(value, [transformation]); - _.set(outputData, sourcePath, transformedValue); - } - } - } - - return outputData; -} - -function applyTransformationsOnValue(value: any, transformations: Transformation[]): any { - // Find all transformations for the given sourcePath - - for (const transformation of transformations) { - switch (transformation.operationType) { - case 'function': - if (typeof transformation.function === 'string') { - try { - // Simple formula evaluator using Function (make sure input is trusted or sandboxed!) - const func = new Function('x', `return ${transformation.function}`); - value = func(value); - } catch (e) { - console.warn(`Failed to evaluate function "${transformation.function}":`, e); - } - } - break; - - case 'valueMapping': - if (transformation.valueMapping) { - const mapped = transformation.valueMapping[value]; - if (mapped !== undefined) { - value = mapped; - } - } - break; - - default: - console.warn(`Unknown transformation type: ${transformation.operationType}`); - } - } - - return value; -} diff --git a/meta_configurator/src/data-mapping/stml/dataMappingSchema.ts b/meta_configurator/src/data-mapping/stml/dataMappingSchema.ts deleted file mode 100644 index 1d6807b7c..000000000 --- a/meta_configurator/src/data-mapping/stml/dataMappingSchema.ts +++ /dev/null @@ -1,120 +0,0 @@ -import type {TopLevelSchema} from '@/schema/jsonSchemaType'; - -export const DATA_MAPPING_SCHEMA: TopLevelSchema = { - $schema: 'http://json-schema.org/draft-07/schema#', - type: 'object', - title: 'Data Mapping Config', - description: 'Configuration for data mapping and transformations.', - properties: { - mappings: { - title: - 'Mappings from source instance to target instance. Any mappings that are not specified here will be ignored and the corresponding values from the source document will not be ported over to the target document', - type: 'array', - items: { - type: 'object', - properties: { - sourcePath: { - type: 'string', - title: 'Path to the source data, a JSON pointer.', - description: - 'The path to the source data in the input JSON document. The path should be a valid JSON pointer, and it can include array indices using the format %INDEX_[A-Z]%. For example, /people/%INDEX_A%/firstName. The %INDEX_A% placeholder represents a path that is an array and the index will be replaced with the actual index of the item in the array during the data conversion.', - pattern: '^#?/[^/]+(/%INDEX_[A-Z]+)?(/[^/]+(/%INDEX_[A-Z]+)*)*$', - }, - targetPath: { - type: 'string', - title: 'Path to the target data, a JSON pointer.', - description: - 'The path to the target data in the output JSON document. The path should be a valid JSON pointer, and it can include array indices using the format %INDEX_[A-Z]%. For example, /person/%INDEX_A%/given_name. The %INDEX_A% placeholder represents a path that is an array and the index will be replaced with the actual index of the item in the array during the data conversion.', - pattern: '^#?/[^/]+(/%INDEX_[A-Z]+)?(/[^/]+(/%INDEX_[A-Z]+)*)*$', - }, - }, - required: ['sourcePath', 'targetPath'], - }, - }, - transformations: { - type: 'array', - title: 'Transformations to apply to the source data before mapping.', - description: - 'Transformations to apply to the source data before mapping. Each transformation will be applied to the source data before the mapping is performed. The transformations are applied in the order they are specified in this array.', - items: { - type: 'object', - properties: { - operationType: { - type: 'string', - enum: ['function', 'valueMapping'], - }, - sourcePath: { - type: 'string', - pattern: '^#?/[^/]+(/%INDEX_[A-Z]+)?(/[^/]+(/%INDEX_[A-Z]+)*)*$', - }, - function: { - type: 'string', - maxLength: 255, - }, - valueMapping: { - type: 'object', - additionalProperties: {}, - }, - }, - required: ['operationType', 'sourcePath'], - anyOf: [ - { - description: 'Any JavaScript function. The original value is provided as variable "x".', - examples: ['x + 1', 'x * 2', 'x.toUpperCase()'], - required: ['function'], - properties: { - operationType: { - const: 'function', - }, - }, - }, - { - required: ['valueMapping'], - properties: { - operationType: { - const: 'valueMapping', - }, - }, - }, - ], - }, - }, - }, - required: ['mappings', 'transformations'], -}; - -export const DATA_MAPPING_EXAMPLE_CONFIG = { - mappings: [ - { - sourcePath: '/people/%INDEX_A%/firstName', - targetPath: '/person/%INDEX_A%/given_name', - }, - { - sourcePath: '/people/%INDEX_A%/lastName', - targetPath: '/person/%INDEX_A%/family_name', - }, - { - sourcePath: '/people/%INDEX_A%/age', - targetPath: 'person/%INDEX_A%/age', - }, - { - sourcePath: '/people/%INDEX_A%/marriageStatus', - targetPath: '/person/%INDEX_A%/married', - }, - ], - transformations: [ - { - operationType: 'mathFormula', - sourcePath: '/people/%INDEX_A%/age', - function: 'x + 1', - }, - { - operationType: 'valueMapping', - sourcePath: '/people/%INDEX_A%/marriageStatus', - valueMapping: { - 0: false, - 1: true, - }, - }, - ], -}; diff --git a/meta_configurator/src/data-mapping/stml/dataMappingServiceStml.ts b/meta_configurator/src/data-mapping/stml/dataMappingServiceStml.ts deleted file mode 100644 index 2ab1030e0..000000000 --- a/meta_configurator/src/data-mapping/stml/dataMappingServiceStml.ts +++ /dev/null @@ -1,177 +0,0 @@ -import type {DataMappingService} from '@/data-mapping/dataMappingService'; -import type {TopLevelSchema} from '@/schema/jsonSchemaType'; -import {inferJsonSchema} from '@/schema/inferJsonSchema'; -import {fixAndParseGeneratedJson, getApiKey} from '@/components/panels/ai-prompts/aiPromptUtils'; -import {queryDataMappingConfig} from '@/utility/ai/aiEndpoint'; -import { - extractInvalidSourcePathsFromConfig, - extractSuitableSourcePaths, -} from '@/data-mapping/stml/extractPathsFromDocument'; -import { - DATA_MAPPING_EXAMPLE_CONFIG, - DATA_MAPPING_SCHEMA, -} from '@/data-mapping/stml/dataMappingSchema'; -import type {DataMappingConfig} from '@/data-mapping/stml/dataMappingTypes'; -import { - normalizeInputConfig, - performSimpleDataMapping, -} from '@/data-mapping/stml/performDataMapping'; -import {ValidationService} from '@/schema/validationService'; -import * as console from 'node:console'; -import {trimDataToMaxSize} from '@/utility/trimData'; - -export class DataMappingServiceStml implements DataMappingService { - async generateMappingSuggestion( - input: any, - targetSchema: TopLevelSchema, - userComments: string - ): Promise<{config: string; success: boolean; message: string}> { - console.log('input is: ', input); - const inputDataSubset = trimDataToMaxSize(input); - console.log( - 'Reduced input data from ' + - JSON.stringify(input).length / 1024 + - ' KB to ' + - JSON.stringify(inputDataSubset).length / 1024 + - ' KB' - ); - - // infer schema for input data - const inputFileSchema = inferJsonSchema(inputDataSubset); - const apiKey = getApiKey(); - const possibleSourcePaths = extractSuitableSourcePaths(input); - - const dataMappingSchemaStr = JSON.stringify(DATA_MAPPING_SCHEMA); - const dataMappingExampleStr = JSON.stringify(DATA_MAPPING_EXAMPLE_CONFIG); - const inputFileSchemaStr = JSON.stringify(inputFileSchema); - const targetSchemaStr = JSON.stringify(targetSchema); - const inputDataSubsetStr = JSON.stringify(inputDataSubset); - console.log( - 'Sizes of the different input files in KB:' + - ' dataMappingSchema: ' + - (dataMappingSchemaStr.length / 1024).toFixed(2) + - ' inputFileSchema: ' + - (inputFileSchemaStr.length / 1024).toFixed(2) + - ' targetSchema: ' + - (targetSchemaStr.length / 1024).toFixed(2) + - ' inputDataSubset: ' + - (inputDataSubsetStr.length / 1024).toFixed(2) - ); - const resultPromise = queryDataMappingConfig( - apiKey, - dataMappingSchemaStr, - dataMappingExampleStr, - inputFileSchemaStr, - targetSchemaStr, - inputDataSubsetStr, - possibleSourcePaths, - userComments - ); - - const responseStr = await resultPromise; - try { - const sanitizedConfig = this.sanitizeMappingConfig(responseStr, input); - return { - config: sanitizedConfig, - success: true, - message: 'Data mapping suggestion generated successfully.', - }; - } catch (e) { - console.error('Error sanitizing mapping config: ', e); - return { - config: responseStr, - success: false, - message: 'Error sanitizing data mapping suggestion: ' + e.message, - }; - } - } - - async performDataMapping( - input: any, - config: string - ): Promise<{resultData: any; success: boolean; message: string}> { - const mapping = JSON.parse(config) as DataMappingConfig; - console.log('parsed mapping is: ', mapping); - try { - const result = performSimpleDataMapping(input, mapping); - return { - resultData: result, - success: true, - message: 'Data mapping performed successfully.', - }; - } catch (e) { - console.error('Error performing data mapping: ', e); - return { - resultData: {}, - success: false, - message: 'Error performing data mapping: ' + e.message, - }; - } - } - - sanitizeInputDocument(input: any): any { - return input; - } - - sanitizeMappingConfig(config: string, input: any): string { - const configObj = fixAndParseGeneratedJson(config); - const configValidated: DataMappingConfig = configObj as DataMappingConfig; - - // normalize - normalizeInputConfig(configObj); - - // remove invalid path mappings or transformations - const invalidUsedSourcePaths = extractInvalidSourcePathsFromConfig(configValidated, input); - if (invalidUsedSourcePaths.length > 0) { - console.log( - `The following source paths are not valid in the input file: ${invalidUsedSourcePaths.join( - ', ' - )}. They will be removed from the configuration.` - ); - } - configValidated.mappings = configValidated.mappings.filter(mapping => { - return !invalidUsedSourcePaths.includes(mapping.sourcePath); - }); - - return JSON.stringify(configObj, null, 2); - } - - validateMappingConfig(config: string, input: any): {success: boolean; message: string} { - const configSchemaValidator = new ValidationService(DATA_MAPPING_SCHEMA); - - let configJson: any; - // parse config to JSON - try { - configJson = JSON.parse(config); - } catch (e) { - return { - success: false, - message: - 'The data mapping configuration is not valid JSON. Please check the syntax and try again.', - }; - } - - const configValidationResult = configSchemaValidator.validate(configJson); - if (configValidationResult.errors.length > 0) { - const formattedErrors = configValidationResult.errors - .map(error => { - return ( - '' + - error.message + - ' at path "' + - error.instancePath + - '" (schema path: "' + - error.schemaPath + - '").' - ); - }) - .join('\n '); - return { - success: false, - message: `The data mapping configuration is invalid: ${formattedErrors}`, - }; // TODO: automated error recovery - } - - return {success: true, message: 'The data mapping configuration is valid.'}; - } -} diff --git a/meta_configurator/src/data-mapping/stml/dataMappingTypes.ts b/meta_configurator/src/data-mapping/stml/dataMappingTypes.ts deleted file mode 100644 index 29df5465e..000000000 --- a/meta_configurator/src/data-mapping/stml/dataMappingTypes.ts +++ /dev/null @@ -1,18 +0,0 @@ -// data mapping config type definitions -export type DataMappingConfig = { - mappings: Mapping[]; - transformations: Transformation[]; -}; - -export type Mapping = { - // paths are JSON pointer paths - sourcePath: string; - targetPath: string; -}; - -export type Transformation = { - operationType: 'function' | 'valueMapping'; - sourcePath: string; - function?: string; - valueMapping?: Record; -}; diff --git a/meta_configurator/src/data-mapping/stml/dataMappingUtilsStml.ts b/meta_configurator/src/data-mapping/stml/dataMappingUtilsStml.ts deleted file mode 100644 index 12118dfe1..000000000 --- a/meta_configurator/src/data-mapping/stml/dataMappingUtilsStml.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type {Path} from '@/utility/path'; -import {jsonPointerToPathTyped} from '@/utility/pathUtils'; - -export function pathToNormalizedJsonPointer( - path: Path, - replaceIndexByPlaceholder: boolean -): string { - let resultPath = path; - if (replaceIndexByPlaceholder) { - // replace all array indices (numbers in the path) by a %INDEX_A%, %INDEX_B%,... placeholder - // first array index is %INDEX_A%, second %INDEX_B%, third %INDEX_C% and so on - let arrayIndex = 0; - resultPath = resultPath.map(seg => { - if (typeof seg === 'number') { - const placeholder = String.fromCharCode(65 + arrayIndex); // 65 is ASCII for 'A' - arrayIndex++; - return `%INDEX_${placeholder}%`; - } - return seg; - }); - } - - // then convert path to json pointer - // note that we do not use the library call here, because the library does not support the %INDEX_X% placeholders - return '/' + resultPath.join('/'); -} - -export function normalizeJsonPointer( - jsonPointer: string, - replaceIndexByPlaceholder: boolean -): string { - // for each path, if it starts with a hashtag, remove the hashtag. In the mappings and also transformations - // if the path starts without a slash '/', then add a '/' at the beginning - if (jsonPointer.startsWith('#')) { - jsonPointer = jsonPointer.slice(1); - } - if (!jsonPointer.startsWith('/')) { - jsonPointer = '/' + jsonPointer; - } - - const pathTyped = jsonPointerToPathTyped(jsonPointer); - return pathToNormalizedJsonPointer(pathTyped, replaceIndexByPlaceholder); -} diff --git a/meta_configurator/src/data-mapping/stml/extractPathsFromDocument.ts b/meta_configurator/src/data-mapping/stml/extractPathsFromDocument.ts deleted file mode 100644 index 20e9cfe94..000000000 --- a/meta_configurator/src/data-mapping/stml/extractPathsFromDocument.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type {Path} from '@/utility/path'; - -import { - normalizeJsonPointer, - pathToNormalizedJsonPointer, -} from '@/data-mapping/stml/dataMappingUtilsStml'; -import type {DataMappingConfig} from '@/data-mapping/stml/dataMappingTypes'; - -export function extractSuitableSourcePaths(inputData: any): string[] { - // finds all leafs in the input json document - const allLeafs = determineAllLeafs(inputData, []); - const allLeafsNormalized = allLeafs.map(leaf => { - return pathToNormalizedJsonPointer(leaf, true); - }); - - // remove duplicates - return Array.from(new Set(allLeafsNormalized)); -} - -export function extractSourcePaths(config: DataMappingConfig): string[] { - const usedSourcePathsMapping = config.mappings.map(mapping => { - return mapping.sourcePath; - }); - const usedSourcePathsTransformations = config.transformations.map(transformation => { - return transformation.sourcePath; - }); - - const uniqueSourcePaths = Array.from( - new Set(usedSourcePathsMapping.concat(usedSourcePathsTransformations)) - ); - - return uniqueSourcePaths.map(jsonPointer => { - return normalizeJsonPointer(jsonPointer, true); - }); -} - -export function extractInvalidSourcePathsFromConfig( - config: DataMappingConfig, - inputData: any -): string[] { - const suitableSourcePaths = extractSuitableSourcePaths(inputData); - const actualSourcePathsInMapping = extractSourcePaths(config); - return actualSourcePathsInMapping.filter(path => !suitableSourcePaths.includes(path)); -} - -function determineAllLeafs(data: any, currentPath: Path): Path[] { - if (Array.isArray(data)) { - return data.flatMap((item, index) => { - const newPath = [...currentPath, index]; - return determineAllLeafs(item, newPath); - }); - } - if (typeof data === 'object' && data !== null) { - return Object.entries(data).flatMap(([key, value]) => { - const newPath = [...currentPath, key]; - return determineAllLeafs(value, newPath); - }); - } - - return [currentPath]; // Leaf node -} diff --git a/meta_configurator/src/data-mapping/stml/findMatchingPaths.ts b/meta_configurator/src/data-mapping/stml/findMatchingPaths.ts deleted file mode 100644 index 53d7be179..000000000 --- a/meta_configurator/src/data-mapping/stml/findMatchingPaths.ts +++ /dev/null @@ -1,46 +0,0 @@ -import _ from 'lodash'; -import type {Path} from '@/utility/path'; - -export function findMatchingPaths(inputData: any, sourcePathDef: string): Path[] { - const allPaths: Path[] = []; - - // Recursively collect all paths to leaf nodes - function collectPaths(obj: any, currentPath: (string | number)[] = []) { - if (_.isPlainObject(obj)) { - for (const key of Object.keys(obj)) { - collectPaths(obj[key], [...currentPath, key]); - } - } else if (Array.isArray(obj)) { - for (let i = 0; i < obj.length; i++) { - collectPaths(obj[i], [...currentPath, i]); - } - } else { - allPaths.push(currentPath); - } - } - - collectPaths(inputData); - - // Normalize the source path definition - const normalizedDef = normalizePathPattern(sourcePathDef); - - // Filter paths that match the pattern - const matchingPointers = allPaths.filter(p => normalizePathArray(p) === normalizedDef); - - // returns a list of Paths instead of JSON pointers - return matchingPointers; -} - -// Replace all array indices (numbers) with "%INDEX%" in the actual path -function normalizePathArray(path: (string | number)[]): string { - return path.map(seg => (typeof seg === 'number' ? '%INDEX%' : seg)).join('/'); -} - -// Replace all $INDEX_X$ with "%INDEX%" in the pattern -function normalizePathPattern(jsonPointer: string): string { - return jsonPointer - .split('/') - .filter(Boolean) - .map(seg => (seg.match(/^%INDEX_[A-Z]%$/) ? '%INDEX%' : seg)) - .join('/'); -} diff --git a/meta_configurator/src/data-mapping/stml/performDataMapping.ts b/meta_configurator/src/data-mapping/stml/performDataMapping.ts deleted file mode 100644 index 3f1067092..000000000 --- a/meta_configurator/src/data-mapping/stml/performDataMapping.ts +++ /dev/null @@ -1,128 +0,0 @@ -import _ from 'lodash'; -import {dataAt} from '@/utility/resolveDataAtPath'; -import {jsonPointerToPathTyped} from '@/utility/pathUtils'; -import type {DataMappingConfig} from '@/data-mapping/stml/dataMappingTypes'; -import {applyTransformations} from '@/data-mapping/stml/applyTransformations'; -import {normalizeJsonPointer} from '@/data-mapping/stml/dataMappingUtilsStml'; - -// Utility Functions -function getIndexPlaceholders(path: string): string[] { - const regex = /%INDEX_([A-Z])%/g; - const matches = new Set(); - let match; - while ((match = regex.exec(path)) !== null) { - matches.add(match[1]); - } - return Array.from(matches); -} - -function resolvePathWithIndexMap(path: string, indexMap: Record): string { - return path.replace(/%INDEX_([A-Z])%/g, (_, p1) => String(indexMap[p1])); -} - -// Recursive Traversal for Nested Indexes -function recursiveMap( - inputData: any, - outputData: any, - sourcePath: string, - targetPath: string, - placeholders: string[], - indexMap: Record, - depth: number -) { - if (depth === placeholders.length) { - const resolvedSource = resolvePathWithIndexMap(sourcePath, indexMap); - const resolvedTarget = resolvePathWithIndexMap(targetPath, indexMap); - try { - const value = dataAt(jsonPointerToPathTyped(resolvedSource), inputData); - if (value !== undefined) { - _.set(outputData, jsonPointerToPathTyped(resolvedTarget), value); - } else { - console.warn(`Skipping mapping: no value at ${resolvedSource}`); - } - } catch (TypeError) { - console.warn(`Error resolving path: ${resolvedSource} for input data `, inputData); - } - return; - } - - const currentPlaceholder = placeholders[depth]; - let arrayPath = sourcePath.split(`%INDEX_${currentPlaceholder}%`)[0]; - // if arrayPath ends with a /, remove it - if (arrayPath.endsWith('/')) { - arrayPath = arrayPath.slice(0, -1); - } - const resolvedArrayPath = resolvePathWithIndexMap(arrayPath, indexMap); - const resolvedArrayPathTyped = jsonPointerToPathTyped(resolvedArrayPath); - const array = dataAt(resolvedArrayPathTyped, inputData); - - if (!Array.isArray(array)) { - console.warn(`Expected array at ${resolvedArrayPath}, got:`, array); - return; - } - - for (let i = 0; i < array.length; i++) { - const newIndexMap = {...indexMap, [currentPlaceholder]: i}; - recursiveMap( - inputData, - outputData, - sourcePath, - targetPath, - placeholders, - newIndexMap, - depth + 1 - ); - } -} - -export function performSimpleDataMapping(inputData: any, mappingConfig: DataMappingConfig): any { - const outputData: any = {}; - - // first, apply the transformations to the complete input data - inputData = applyTransformations(inputData, mappingConfig.transformations); - - for (const mapping of mappingConfig.mappings) { - const {sourcePath, targetPath} = mapping; - const placeholders = getIndexPlaceholders(sourcePath + targetPath); - - if (placeholders.length === 0) { - const value = dataAt(jsonPointerToPathTyped(sourcePath), inputData); - if (value !== undefined) { - const targetPathTyped = jsonPointerToPathTyped(targetPath); - _.set(outputData, targetPathTyped, value); - } else { - console.warn(`Skipping mapping: no value at ${sourcePath}`); - } - } else { - recursiveMap(inputData, outputData, sourcePath, targetPath, placeholders, {}, 0); - } - } - - // for the rare scenario of having an array at root level, we apply this transformation - return turnArrayLikeObjectIntoArray(outputData); -} - -function turnArrayLikeObjectIntoArray(obj: any): any | any[] { - // if the object has only keys that are numbers from 1 to n, turn it into an array, keeping the order - if (typeof obj === 'object' && obj !== null) { - const keys = Object.keys(obj) - .map(Number) - .sort((a, b) => a - b); - // check also that there is no gap in the keys - if (keys.length > 0 && keys[0] === 0 && keys.every((key, index) => key === index)) { - return keys.map(key => obj[key]); - } - } - - return obj; -} - -export function normalizeInputConfig(inputConfig: DataMappingConfig) { - inputConfig.mappings.forEach(mapping => { - mapping.sourcePath = normalizeJsonPointer(mapping.sourcePath, false); - mapping.targetPath = normalizeJsonPointer(mapping.targetPath, false); - }); - inputConfig.transformations.forEach(transformation => { - transformation.sourcePath = normalizeJsonPointer(transformation.sourcePath, false); - }); -} diff --git a/meta_configurator/src/main.ts b/meta_configurator/src/main.ts index 934485a56..764167790 100644 --- a/meta_configurator/src/main.ts +++ b/meta_configurator/src/main.ts @@ -7,7 +7,6 @@ import Tooltip from 'primevue/tooltip'; import ToastService from 'primevue/toastservice'; import ConfirmationService from 'primevue/confirmationservice'; import {useAppRouter} from './router/router'; -import ErrorService from '@/utility/errorService'; import {registerIcons} from '@/fontawesome'; import {registerDefaultDataFormats} from '@/dataformats/defaultFormats'; @@ -17,6 +16,7 @@ import {SessionMode} from '@/store/sessionMode'; import {registerDefaultPanelTypes} from '@/components/panels/defaultPanelTypes'; import {definePreset} from '@primevue/themes'; import {initErrorService, useErrorService} from '@/utility/errorServiceInstance'; +import {initApiKey} from '@/utility/ai/apiKey'; // @ts-ignore const app = createApp(App); @@ -90,6 +90,7 @@ app.config.errorHandler = (error: unknown) => useErrorService().onError(error); registerIcons(); registerDefaultDataFormats(); registerDefaultPanelTypes(); +initApiKey(); // warn the user if he closes the app window.addEventListener('beforeunload', event => { diff --git a/meta_configurator/src/schema/__tests__/detectSchemaFeatures.test.ts b/meta_configurator/src/schema/__tests__/detectSchemaFeatures.test.ts new file mode 100644 index 000000000..12dc25f7a --- /dev/null +++ b/meta_configurator/src/schema/__tests__/detectSchemaFeatures.test.ts @@ -0,0 +1,163 @@ +import {describe, expect, it, vi} from 'vitest'; +import {detectSchemaFeatures} from '@/schema/detectSchemaFeatures'; +import type {TopLevelSchema} from '@/schema/jsonSchemaType'; + +vi.mock('@/dataformats/formatRegistry', () => ({ + useDataConverter: () => ({ + stringify: (data: any) => JSON.stringify(data), + parse: (data: string) => JSON.parse(data), + }), +})); + +describe('detectSchemaFeatures', () => { + it('detects no features in a minimal schema', () => { + const schema: TopLevelSchema = {type: 'object'}; + const features = detectSchemaFeatures(schema); + + expect(features.composition).toBe(false); + expect(features.conditionals).toBe(false); + expect(features.defaultValues).toBe(false); + expect(features.exampleValues).toBe(false); + expect(features.enums).toBe(false); + expect(features.constants).toBe(false); + expect(features.multipleTypes).toBe(false); + expect(features.references).toBe(false); + expect(features.required).toBe(false); + expect(features.negation).toBe(false); + expect(features.booleanSchemas).toBe(false); + expect(features.descriptions).toBe(false); + expect(features.titles).toBe(false); + expect(features.externalReferences).toBe(false); + }); + + it('detects composition (allOf/anyOf/oneOf)', () => { + expect(detectSchemaFeatures({allOf: [{type: 'string'}]}).composition).toBe(true); + expect(detectSchemaFeatures({anyOf: [{type: 'string'}]}).composition).toBe(true); + expect(detectSchemaFeatures({oneOf: [{type: 'string'}, {type: 'number'}]}).composition).toBe( + true + ); + }); + + it('detects conditionals (if/then/else)', () => { + expect(detectSchemaFeatures({if: {type: 'string'}, then: {minLength: 1}}).conditionals).toBe( + true + ); + expect(detectSchemaFeatures({then: {minLength: 1}}).conditionals).toBe(true); + expect(detectSchemaFeatures({else: true}).conditionals).toBe(true); + }); + + it('detects default values including falsy ones', () => { + expect(detectSchemaFeatures({default: 'hello'}).defaultValues).toBe(true); + expect(detectSchemaFeatures({default: 0}).defaultValues).toBe(true); + expect(detectSchemaFeatures({default: false}).defaultValues).toBe(true); + expect(detectSchemaFeatures({default: null}).defaultValues).toBe(true); + expect(detectSchemaFeatures({default: ''}).defaultValues).toBe(true); + }); + + it('detects example values', () => { + expect(detectSchemaFeatures({examples: ['foo']}).exampleValues).toBe(true); + }); + + it('detects enums', () => { + expect(detectSchemaFeatures({enum: ['a', 'b']}).enums).toBe(true); + }); + + it('detects constants including falsy ones', () => { + expect(detectSchemaFeatures({const: 'foo'}).constants).toBe(true); + expect(detectSchemaFeatures({const: 0}).constants).toBe(true); + expect(detectSchemaFeatures({const: false}).constants).toBe(true); + expect(detectSchemaFeatures({const: null}).constants).toBe(true); + }); + + it('detects multiple types', () => { + expect(detectSchemaFeatures({type: ['string', 'null']}).multipleTypes).toBe(true); + expect(detectSchemaFeatures({type: 'string'}).multipleTypes).toBe(false); + }); + + it('detects $ref references', () => { + const schema: TopLevelSchema = { + type: 'object', + properties: { + foo: {$ref: '#/$defs/bar'}, + }, + $defs: {bar: {type: 'string'}}, + }; + expect(detectSchemaFeatures(schema).references).toBe(true); + expect(detectSchemaFeatures(schema).externalReferences).toBe(false); + }); + + it('detects external references', () => { + const schema: TopLevelSchema = {$ref: 'https://example.com/schema.json'}; + expect(detectSchemaFeatures(schema).references).toBe(true); + expect(detectSchemaFeatures(schema).externalReferences).toBe(true); + }); + + it('detects required', () => { + expect(detectSchemaFeatures({type: 'object', required: ['name']}).required).toBe(true); + }); + + it('detects negation (not)', () => { + expect(detectSchemaFeatures({not: {type: 'string'}}).negation).toBe(true); + }); + + it('detects boolean schemas', () => { + const schema: TopLevelSchema = { + type: 'object', + properties: { + anything: true, + nothing: false, + }, + } as TopLevelSchema; + expect(detectSchemaFeatures(schema).booleanSchemas).toBe(true); + }); + + it('detects descriptions', () => { + expect(detectSchemaFeatures({description: 'A schema'}).descriptions).toBe(true); + }); + + it('detects titles', () => { + expect(detectSchemaFeatures({title: 'My Schema'}).titles).toBe(true); + }); + + it('detects features in nested schemas', () => { + const schema: TopLevelSchema = { + type: 'object', + properties: { + nested: { + type: 'object', + properties: { + deep: { + enum: ['a', 'b'], + default: 'a', + description: 'A deep field', + }, + }, + }, + }, + }; + const features = detectSchemaFeatures(schema); + expect(features.enums).toBe(true); + expect(features.defaultValues).toBe(true); + expect(features.descriptions).toBe(true); + }); + + it('detects features inside $defs', () => { + const schema: TopLevelSchema = { + $defs: { + myType: { + type: 'string', + title: 'My Type', + allOf: [{minLength: 1}], + }, + }, + }; + const features = detectSchemaFeatures(schema); + expect(features.titles).toBe(true); + expect(features.composition).toBe(true); + }); + + it('does not fail on malformed schema in lenient mode (default)', () => { + const schema = {type: 123} as any; + expect(() => detectSchemaFeatures(schema)).not.toThrow(); + }); +}); diff --git a/meta_configurator/src/schema/__tests__/jsonSchemaVisitor.test.ts b/meta_configurator/src/schema/__tests__/jsonSchemaVisitor.test.ts new file mode 100644 index 000000000..be775e5fb --- /dev/null +++ b/meta_configurator/src/schema/__tests__/jsonSchemaVisitor.test.ts @@ -0,0 +1,325 @@ +import {describe, expect, it} from 'vitest'; +import type {JsonSchemaObjectType, JsonSchemaType} from '@/schema/jsonSchemaType'; +import {JsonSchemaVisitor, type VisitorContext} from '@/schema/jsonSchemaVisitor'; + +class RecordingVisitor extends JsonSchemaVisitor { + readonly schemas: Array<{schema: JsonSchemaObjectType; ctx: VisitorContext}> = []; + readonly booleans: Array<{value: boolean; ctx: VisitorContext}> = []; + readonly refs: Array<{ref: string; ctx: VisitorContext}> = []; + readonly properties: Array<{name: string; schema: JsonSchemaType; ctx: VisitorContext}> = []; + readonly patternProperties: Array<{ + pattern: string; + schema: JsonSchemaType; + ctx: VisitorContext; + }> = []; + readonly subSchemaKeywords: Array<{ + keyword: string; + schema: JsonSchemaType; + ctx: VisitorContext; + }> = []; + readonly compositionals: Array<{ + keyword: string; + schemas: JsonSchemaType | JsonSchemaType[]; + ctx: VisitorContext; + }> = []; + readonly conditionals: Array<{keyword: string; schema: JsonSchemaType; ctx: VisitorContext}> = []; + readonly definitions: Array<{name: string; schema: JsonSchemaType; ctx: VisitorContext}> = []; + + protected visitSchema(schema: JsonSchemaObjectType, ctx: VisitorContext): void { + this.schemas.push({schema, ctx}); + } + protected visitBooleanSchema(value: boolean, ctx: VisitorContext): void { + this.booleans.push({value, ctx}); + } + protected visitRef(ref: string, ctx: VisitorContext): void { + this.refs.push({ref, ctx}); + } + protected visitProperty(name: string, schema: JsonSchemaType, ctx: VisitorContext): void { + this.properties.push({name, schema, ctx}); + } + protected visitPatternProperty( + pattern: string, + schema: JsonSchemaType, + ctx: VisitorContext + ): void { + this.patternProperties.push({pattern, schema, ctx}); + } + protected visitSubSchemaKeyword( + keyword: string, + schema: JsonSchemaType, + ctx: VisitorContext + ): void { + this.subSchemaKeywords.push({keyword, schema, ctx}); + } + protected visitCompositional( + keyword: string, + schemas: JsonSchemaType | JsonSchemaType[], + ctx: VisitorContext + ): void { + this.compositionals.push({keyword, schemas, ctx}); + } + protected visitConditional(keyword: string, schema: JsonSchemaType, ctx: VisitorContext): void { + this.conditionals.push({keyword, schema, ctx}); + } + protected visitDefinition(name: string, schema: JsonSchemaType, ctx: VisitorContext): void { + this.definitions.push({name, schema, ctx}); + } +} + +describe('JsonSchemaVisitor', () => { + it('visits a simple object schema', () => { + const visitor = new RecordingVisitor(); + const schema = {type: 'object', title: 'Root'}; + visitor.traverse(schema); + + expect(visitor.schemas).toHaveLength(1); + expect(visitor.schemas[0].schema).toBe(schema); + expect(visitor.schemas[0].ctx.depth).toBe(0); + expect(visitor.schemas[0].ctx.path).toEqual([]); + expect(visitor.schemas[0].ctx.parentKind).toBeNull(); + }); + + it('visits a boolean schema', () => { + const visitor = new RecordingVisitor(); + visitor.traverse(true); + + expect(visitor.booleans).toHaveLength(1); + expect(visitor.booleans[0].value).toBe(true); + }); + + it('visits properties with correct context', () => { + const visitor = new RecordingVisitor(); + const schema = { + type: 'object', + properties: { + name: {type: 'string'}, + age: {type: 'number'}, + }, + }; + visitor.traverse(schema); + + expect(visitor.properties).toHaveLength(2); + expect(visitor.properties[0].name).toBe('name'); + expect(visitor.properties[0].ctx.path).toEqual(['properties', 'name']); + expect(visitor.properties[0].ctx.parentKind).toBe('property'); + expect(visitor.properties[1].name).toBe('age'); + expect(visitor.properties[1].ctx.path).toEqual(['properties', 'age']); + }); + + it('visits patternProperties', () => { + const visitor = new RecordingVisitor(); + const schema = { + type: 'object', + patternProperties: { + '^S_': {type: 'string'}, + }, + }; + visitor.traverse(schema); + + expect(visitor.patternProperties).toHaveLength(1); + expect(visitor.patternProperties[0].pattern).toBe('^S_'); + expect(visitor.patternProperties[0].ctx.parentKind).toBe('pattern'); + expect(visitor.patternProperties[0].ctx.path).toEqual(['patternProperties', '^S_']); + }); + + it('visits $ref', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({$ref: '#/$defs/foo'}); + + expect(visitor.refs).toHaveLength(1); + expect(visitor.refs[0].ref).toBe('#/$defs/foo'); + }); + + it('visits allOf, anyOf, oneOf', () => { + const visitor = new RecordingVisitor(); + const schema = { + allOf: [{type: 'object'}], + anyOf: [{type: 'string'}, {type: 'number'}], + oneOf: [{type: 'boolean'}], + }; + visitor.traverse(schema); + + const keywords = visitor.compositionals.map(c => c.keyword); + expect(keywords).toContain('allOf'); + expect(keywords).toContain('anyOf'); + expect(keywords).toContain('oneOf'); + }); + + it('visits not', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({not: {type: 'string'}}); + + expect(visitor.compositionals).toHaveLength(1); + expect(visitor.compositionals[0].keyword).toBe('not'); + }); + + it('visits if/then/else', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({ + if: {type: 'string'}, + then: {minLength: 1}, + else: true, + }); + + const conditionalKeywords = visitor.conditionals.map(c => c.keyword); + expect(conditionalKeywords).toContain('if'); + expect(conditionalKeywords).toContain('then'); + expect(conditionalKeywords).toContain('else'); + expect(visitor.conditionals[2].schema).toBe(true); + }); + + it('visits $defs', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({ + $defs: { + foo: {type: 'string'}, + bar: {type: 'number'}, + }, + }); + + expect(visitor.definitions).toHaveLength(2); + const names = visitor.definitions.map(d => d.name); + expect(names).toContain('foo'); + expect(names).toContain('bar'); + expect(visitor.definitions[0].ctx.parentKind).toBe('definition'); + expect(visitor.definitions[0].ctx.path).toEqual(['$defs', 'foo']); + }); + + it('visits legacy definitions keyword', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({definitions: {myDef: {type: 'object'}}}); + + expect(visitor.definitions).toHaveLength(1); + expect(visitor.definitions[0].name).toBe('myDef'); + expect(visitor.definitions[0].ctx.path).toEqual(['definitions', 'myDef']); + }); + + it('visits items as single schema', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({type: 'array', items: {type: 'string'}}); + + expect(visitor.subSchemaKeywords).toHaveLength(1); + expect(visitor.subSchemaKeywords[0].keyword).toBe('items'); + expect(visitor.subSchemaKeywords[0].ctx.path).toEqual(['items']); + }); + + it('visits items as array (tuple)', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({type: 'array', items: [{type: 'string'}, {type: 'number'}]}); + + expect(visitor.subSchemaKeywords).toHaveLength(2); + expect(visitor.subSchemaKeywords[0].ctx.path).toEqual(['items', 0]); + expect(visitor.subSchemaKeywords[1].ctx.path).toEqual(['items', 1]); + }); + + it('visits additionalProperties', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({type: 'object', additionalProperties: {type: 'string'}}); + + expect(visitor.subSchemaKeywords.some(k => k.keyword === 'additionalProperties')).toBe(true); + }); + + it('tracks depth correctly', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({ + type: 'object', + properties: { + child: {type: 'string'}, + }, + }); + + const rootSchema = visitor.schemas.find(s => s.ctx.depth === 0); + const childSchema = visitor.schemas.find(s => s.ctx.depth === 1); + expect(rootSchema).toBeDefined(); + expect(childSchema).toBeDefined(); + expect(childSchema!.ctx.path).toEqual(['properties', 'child']); + }); + + it('visits deeply nested schemas', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({ + type: 'object', + properties: { + outer: { + type: 'object', + properties: { + inner: {type: 'string'}, + }, + }, + }, + }); + + const innerProp = visitor.properties.find(p => p.name === 'inner'); + expect(innerProp).toBeDefined(); + expect(innerProp!.ctx.path).toEqual(['properties', 'outer', 'properties', 'inner']); + expect(innerProp!.ctx.depth).toBe(2); + }); + + it('strict mode throws on invalid keyword value', () => { + const visitor = new RecordingVisitor(true); + expect(() => visitor.traverse({type: 123 as any})).toThrow(TypeError); + }); + + it('lenient mode skips invalid keyword value', () => { + const visitor = new RecordingVisitor(false); + expect(() => visitor.traverse({type: 123 as any})).not.toThrow(); + }); + + it('strict mode throws on non-schema value', () => { + const visitor = new RecordingVisitor(true); + expect(() => visitor.traverse('not a schema' as any)).toThrow(TypeError); + }); + + it('lenient mode skips non-schema value', () => { + const visitor = new RecordingVisitor(false); + expect(() => visitor.traverse('not a schema' as any)).not.toThrow(); + }); + + it('visits dependentSchemas as definitions', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({ + dependentSchemas: { + credit_card: {required: ['billing_address']}, + }, + }); + + expect(visitor.definitions).toHaveLength(1); + expect(visitor.definitions[0].name).toBe('credit_card'); + expect(visitor.definitions[0].ctx.path).toEqual(['dependentSchemas', 'credit_card']); + }); + + it('skips array-valued dependencies, visits schema-valued ones', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({ + dependencies: { + name: ['address'], + address: {required: ['city']}, + }, + } as any); + + expect(visitor.definitions).toHaveLength(1); + expect(visitor.definitions[0].name).toBe('address'); + }); + + it('visits $dynamicRef and $recursiveRef', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({$dynamicRef: '#myAnchor'} as any); + expect(visitor.refs).toHaveLength(1); + expect(visitor.refs[0].ref).toBe('#myAnchor'); + + const visitor2 = new RecordingVisitor(); + visitor2.traverse({$recursiveRef: '#'} as any); + expect(visitor2.refs).toHaveLength(1); + expect(visitor2.refs[0].ref).toBe('#'); + }); + + it('handles prefixItems', () => { + const visitor = new RecordingVisitor(); + visitor.traverse({type: 'array', prefixItems: [{type: 'string'}, {type: 'number'}]} as any); + + const prefixItemKeywords = visitor.subSchemaKeywords.filter(k => k.keyword === 'prefixItems'); + expect(prefixItemKeywords).toHaveLength(2); + expect(prefixItemKeywords[0].ctx.path).toEqual(['prefixItems', 0]); + expect(prefixItemKeywords[1].ctx.path).toEqual(['prefixItems', 1]); + }); +}); diff --git a/meta_configurator/src/schema/detectSchemaFeatures.ts b/meta_configurator/src/schema/detectSchemaFeatures.ts index 597c19e4c..e93840db2 100644 --- a/meta_configurator/src/schema/detectSchemaFeatures.ts +++ b/meta_configurator/src/schema/detectSchemaFeatures.ts @@ -1,5 +1,6 @@ -import type {JsonSchemaObjectType, JsonSchemaType, TopLevelSchema} from '@/schema/jsonSchemaType'; +import type {JsonSchemaObjectType, TopLevelSchema} from '@/schema/jsonSchemaType'; import {isExternalRef} from '@/schema/externalReferences.ts'; +import {JsonSchemaVisitor} from '@/schema/jsonSchemaVisitor.ts'; export type SchemaFeatures = { composition: boolean; @@ -19,10 +20,10 @@ export type SchemaFeatures = { // on purpose, we currently do not track whether it uses constraints (e.g., maxLength, minLength, etc.) because there are so many of them and for our use case we do not need to know about them }; -export function detectSchemaFeatures(jsonSchema: TopLevelSchema): SchemaFeatures { - const features: SchemaFeatures = { - composition: false, // oneOf, anyOf, allOf - conditionals: false, // if, then, else +class SchemaFeaturesVisitor extends JsonSchemaVisitor { + readonly features: SchemaFeatures = { + composition: false, + conditionals: false, defaultValues: false, exampleValues: false, enums: false, @@ -31,151 +32,39 @@ export function detectSchemaFeatures(jsonSchema: TopLevelSchema): SchemaFeatures references: false, required: false, negation: false, - booleanSchemas: false, // when a sub-schema is simply true or false instead of an object + booleanSchemas: false, descriptions: false, titles: false, - externalReferences: false, // whether the schema contains $refs that point to external schemas (i.e., not starting with #) + externalReferences: false, }; - // detect features for root level schema - detectSubSchemaFeatures(jsonSchema, features); - - // detect features for all $defs and definitions - if (jsonSchema.$defs) { - for (const key in jsonSchema.$defs) { - detectSubSchemaFeatures(jsonSchema.$defs[key], features); - } - } - if (jsonSchema.definitions) { - for (const key in jsonSchema.definitions) { - detectSubSchemaFeatures(jsonSchema.definitions[key], features); - } + protected visitBooleanSchema(): void { + this.features.booleanSchemas = true; } - return features; -} - -function detectSubSchemaPropertiesFeatures(subSchema: JsonSchemaType, features: SchemaFeatures) { - // the sub schema will be either true, false or a dictionary of keys that map to a sub schema - // this function will be called for properties, patternProperties each - if (typeof subSchema === 'object' && !Array.isArray(subSchema)) { - for (const key in subSchema) { - detectSubSchemaFeatures(subSchema[key], features); - } - } else if (subSchema === true || subSchema === false) { - features.booleanSchemas = true; + protected visitSchema(schema: JsonSchemaObjectType): void { + if (schema.allOf || schema.anyOf || schema.oneOf) this.features.composition = true; + if (schema.if !== undefined || schema.then !== undefined || schema.else !== undefined) + this.features.conditionals = true; + if ('default' in schema) this.features.defaultValues = true; + if (schema.examples) this.features.exampleValues = true; + if (schema.enum) this.features.enums = true; + if ('const' in schema) this.features.constants = true; + if (Array.isArray(schema.type)) this.features.multipleTypes = true; + if (schema.required) this.features.required = true; + if (schema.not !== undefined) this.features.negation = true; + if (schema.description) this.features.descriptions = true; + if (schema.title) this.features.titles = true; } -} -function detectSubSchemaFeatures(subSchema: JsonSchemaType, features: SchemaFeatures) { - if (subSchema == true || subSchema == false) { - features.booleanSchemas = true; - } else { - detectSubObjectSchemaFeatures(subSchema, features); + protected visitRef(ref: string): void { + this.features.references = true; + if (isExternalRef(ref)) this.features.externalReferences = true; } } -function detectSubObjectSchemaFeatures(subSchema: JsonSchemaObjectType, features: SchemaFeatures) { - // Check for composition - if ('oneOf' in subSchema || 'anyOf' in subSchema || 'allOf' in subSchema) { - features.composition = true; - - if ('oneOf' in subSchema) { - detectSchemaFeatures(subSchema.oneOf); - } - if ('anyOf' in subSchema) { - detectSchemaFeatures(subSchema.anyOf); - } - if ('allOf' in subSchema) { - detectSchemaFeatures(subSchema.allOf); - } - } - - // Check for conditionals - if ('if' in subSchema || 'then' in subSchema || 'else' in subSchema) { - features.conditionals = true; - - if ('if' in subSchema) { - detectSubSchemaFeatures(subSchema.if, features); - } - if ('then' in subSchema) { - detectSubSchemaFeatures(subSchema.then, features); - } - if ('else' in subSchema) { - detectSubSchemaFeatures(subSchema.else, features); - } - } - - // Check for default values - if (subSchema.default) { - features.defaultValues = true; - } - - // Check for example values - if (subSchema.examples) { - features.exampleValues = true; - } - - // Check for enums - if (subSchema.enum) { - features.enums = true; - } - - // Check for constants - if (subSchema.const) { - features.constants = true; - } - - // Check for multiple types - if (Array.isArray(subSchema.type)) { - features.multipleTypes = true; - } - - // Check for references - if (subSchema.$ref) { - features.references = true; - if (isExternalRef(subSchema.$ref)) { - features.externalReferences = true; - } - } - - // Check for required properties - if (subSchema.required) { - features.required = true; - } - - // Check for negation - if (subSchema.not) { - features.negation = true; - } - - // Check for descriptions - if (subSchema.description) { - features.descriptions = true; - } - - // Check for titles - if (subSchema.title) { - features.titles = true; - } - - // recursively check for features in properties and patternProperties and additionalProperties - if ('properties' in subSchema) { - detectSubSchemaPropertiesFeatures(subSchema.properties, features); - } - if ('patternProperties' in subSchema) { - detectSubSchemaPropertiesFeatures(subSchema.patternProperties, features); - } - if ('additionalProperties' in subSchema) { - detectSubSchemaFeatures(subSchema.additionalProperties, features); - } - - // recursively check for items in case of arrays - if (Array.isArray(subSchema.items)) { - subSchema.items.forEach(item => { - detectSubSchemaFeatures(item, features); - }); - } else if (subSchema.items) { - detectSubSchemaFeatures(subSchema.items, features); - } +export function detectSchemaFeatures(jsonSchema: TopLevelSchema): SchemaFeatures { + const visitor = new SchemaFeaturesVisitor(false); + visitor.traverse(jsonSchema); + return visitor.features; } diff --git a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorArrays.test.ts b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorArrays.test.ts index d35d1c878..17ee72e49 100644 --- a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorArrays.test.ts +++ b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorArrays.test.ts @@ -7,6 +7,8 @@ import { generateObjectFallbackDisplayName, identifyAllObjects, isSchemaThatDeservesANode, + populateGraph, + trimGraph, } from '../schemaGraphConstructor'; vi.mock('@/dataformats/formatRegistry', () => ({ @@ -253,6 +255,46 @@ describe('test schema graph constructor with objects and attributes, without adv ).toEqual('person'); }); + it('array of array pointing to a referenced object should create an edge to that object', () => { + const nestedArraySchema: TopLevelSchema = { + type: 'object', + $defs: { + Row: { + type: 'object', + properties: { + value: {type: 'number'}, + }, + }, + }, + properties: { + matrix: { + type: 'array', + items: { + type: 'array', + items: {$ref: '#/$defs/Row'}, + }, + }, + }, + }; + + const nestedDefs = identifyAllObjects(nestedArraySchema); + const nestedGraph = new SchemaGraph([], []); + populateGraph(nestedDefs, nestedGraph); + trimGraph(nestedGraph); + + const rootNode = nestedGraph.nodes.find(n => n.absolutePath.length === 0); + expect(rootNode).toBeDefined(); + + const rowNode = nestedGraph.nodes.find(n => n.name === 'Row'); + expect(rowNode).toBeDefined(); + + // There must be an edge from root to Row driven by the nested-array property + const edgeToRow = nestedGraph.edges.find(e => e.start === rootNode && e.end === rowNode); + expect(edgeToRow).toBeDefined(); + expect(edgeToRow?.isArray).toBe(true); + expect(edgeToRow?.label).toBe('matrix'); + }); + it('generate attribute edges', () => { for (const node of defs.values()) { node.attributes = generateObjectAttributes(node.absolutePath, node.schema, defs); diff --git a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorBasicFeatures.test.ts b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorBasicFeatures.test.ts index 356febafb..2fe3424c3 100644 --- a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorBasicFeatures.test.ts +++ b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorBasicFeatures.test.ts @@ -1,12 +1,11 @@ import {beforeEach, describe, expect, it, vi} from 'vitest'; -import type {Path} from '@/utility/path'; import type {TopLevelSchema} from '@/schema/jsonSchemaType'; import {EdgeType, SchemaGraph, SchemaObjectNodeData} from '../schemaGraphTypes'; import { generateAttributeEdges, generateObjectAttributes, generateObjectFallbackDisplayName, - identifyObjects, + identifyAllObjects, isSchemaThatDeservesANode, } from '../schemaGraphConstructor'; @@ -65,12 +64,7 @@ describe('test schema graph constructor with objects and attributes, without adv let defs: Map; beforeEach(() => { - currentPath = []; - defs = new Map(); - - identifyObjects(currentPath, schema, defs, false, schema); - // @ts-ignore - identifyObjects(['$defs', 'person'], schema.$defs.person, defs); + defs = identifyAllObjects(schema); }); it('identify objects', () => { diff --git a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorComposition.test.ts b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorComposition.test.ts index efbeea8dd..6101a2849 100644 --- a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorComposition.test.ts +++ b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorComposition.test.ts @@ -1,5 +1,4 @@ import {beforeEach, describe, expect, it, vi} from 'vitest'; -import type {Path} from '@/utility/path'; import type {TopLevelSchema} from '@/schema/jsonSchemaType'; import {EdgeType, SchemaGraph, SchemaObjectNodeData} from '../schemaGraphTypes'; import { @@ -7,7 +6,7 @@ import { generateObjectAttributes, generateObjectSpecialPropertyEdges, generateObjectFallbackDisplayName, - identifyObjects, + identifyAllObjects, isObjectSchema, isSchemaThatDeservesANode, } from '../schemaGraphConstructor'; @@ -20,7 +19,6 @@ vi.mock('@/dataformats/formatRegistry', () => ({ })); describe('test schema graph constructor with objects and compositional keywords', () => { - let currentPath: Path; let schema: TopLevelSchema = { type: 'object', required: ['propertyObject'], @@ -115,14 +113,7 @@ describe('test schema graph constructor with objects and compositional keywords' let defs: Map; beforeEach(() => { - currentPath = []; - defs = new Map(); - - identifyObjects(currentPath, schema, defs, false, schema); - // @ts-ignore - for (const [key, value] of Object.entries(schema.$defs)) { - identifyObjects(['$defs', key], value, defs, true, schema); - } + defs = identifyAllObjects(schema); }); it('identify objects', () => { diff --git a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorConditionals.test.ts b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorConditionals.test.ts index 94d9af5b3..580dc1ff4 100644 --- a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorConditionals.test.ts +++ b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorConditionals.test.ts @@ -1,12 +1,11 @@ import {beforeEach, describe, expect, it, vi} from 'vitest'; -import type {Path} from '@/utility/path'; import type {TopLevelSchema} from '@/schema/jsonSchemaType'; import {EdgeType, SchemaGraph, SchemaObjectNodeData} from '../schemaGraphTypes'; import { generateObjectAttributes, generateObjectSpecialPropertyEdges, generateObjectFallbackDisplayName, - identifyObjects, + identifyAllObjects, isObjectSchema, populateGraph, trimGraph, @@ -20,7 +19,6 @@ vi.mock('@/dataformats/formatRegistry', () => ({ })); describe('test schema graph constructor with conditionals', () => { - let currentPath: Path; let schema: TopLevelSchema = { type: 'object', required: ['propertyObject'], @@ -66,14 +64,7 @@ describe('test schema graph constructor with conditionals', () => { let defs: Map; beforeEach(() => { - currentPath = []; - defs = new Map(); - - identifyObjects(currentPath, schema, defs, false, schema); - // @ts-ignore - for (const [key, value] of Object.entries(schema.$defs)) { - identifyObjects(['$defs', key], value, defs, true, schema); - } + defs = identifyAllObjects(schema); }); it('identify objects', () => { diff --git a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorEnum.test.ts b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorEnum.test.ts index 2770a0d04..8160b5ad5 100644 --- a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorEnum.test.ts +++ b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorEnum.test.ts @@ -1,11 +1,10 @@ import {beforeEach, describe, expect, it, vi} from 'vitest'; -import type {Path} from '@/utility/path'; import type {TopLevelSchema} from '@/schema/jsonSchemaType'; import {EdgeType, SchemaEnumNodeData, SchemaGraph, SchemaObjectNodeData} from '../schemaGraphTypes'; import { generateAttributeEdges, generateObjectAttributes, - identifyObjects, + identifyAllObjects, populateGraph, trimNodeChildren, } from '../schemaGraphConstructor'; @@ -30,7 +29,6 @@ vi.mock('@/dataformats/formatRegistry', () => ({ })); describe('test schema graph constructor with objects and attributes with enums', () => { - let currentPath: Path; let schema: TopLevelSchema = { type: 'object', $defs: { @@ -64,13 +62,7 @@ describe('test schema graph constructor with objects and attributes with enums', let defs: Map; beforeEach(() => { - currentPath = []; - defs = new Map(); - identifyObjects(currentPath, schema, defs, false, schema); - // @ts-ignore - for (const [key, value] of Object.entries(schema.$defs)) { - identifyObjects(['$defs', key], value, defs, true, schema); - } + defs = identifyAllObjects(schema); }); it('identify objects', () => { diff --git a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorSpecialCases.test.ts b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorSpecialCases.test.ts index 113ce2e93..a18d719ff 100644 --- a/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorSpecialCases.test.ts +++ b/meta_configurator/src/schema/graph-representation/__tests__/schemaGraphConstructorSpecialCases.test.ts @@ -80,6 +80,50 @@ describe('tests for more difficult scenarios and special cases that result as a expect(graph.edges.length).toBe(2); // one edge from the root node to the array entry node and one edge from the array entry node to the enum node }); + it('$defs node with both $ref and properties should have edge from that node to the referenced schema', () => { + const schema: TopLevelSchema = { + type: 'object', + $defs: { + Quantity: { + type: 'object', + properties: { + Value: {type: 'number'}, + Unit: {type: 'string'}, + }, + }, + Time: { + title: 'Time', + $ref: '#/$defs/Quantity', + properties: { + Value: {minimum: 0, type: 'number'}, + Unit: {type: 'string'}, + }, + required: ['Unit', 'Value'], + }, + }, + properties: { + duration: {$ref: '#/$defs/Time'}, + }, + }; + + const defs = identifyAllObjects(schema); + const graph = new SchemaGraph([], []); + populateGraph(defs, graph); + trimGraph(graph); + + const timeNode = graph.nodes.find(n => n.name === 'Time'); + expect(timeNode).toBeDefined(); + + const quantityNode = graph.nodes.find(n => n.name === 'Quantity'); + expect(quantityNode).toBeDefined(); + + // Time node must have an edge pointing to Quantity (via its $ref) + const edgeTimeToQuantity = graph.edges.find( + e => e.start === timeNode && e.end === quantityNode + ); + expect(edgeTimeToQuantity).toBeDefined(); + }); + it('object property has a reference but also defines its own things and both nodes (referenced sub-schema and own definition of sub-schema) should be connected with an edge', () => { let schema: TopLevelSchema = { $defs: { diff --git a/meta_configurator/src/schema/graph-representation/schemaGraphConstructor.ts b/meta_configurator/src/schema/graph-representation/schemaGraphConstructor.ts index 2836b46cc..40c19431f 100644 --- a/meta_configurator/src/schema/graph-representation/schemaGraphConstructor.ts +++ b/meta_configurator/src/schema/graph-representation/schemaGraphConstructor.ts @@ -22,6 +22,7 @@ import { } from '@/schema/graph-representation/schemaGraphTypes'; import {useErrorService} from '@/utility/errorServiceInstance'; import {isExternalRef} from '@/schema/externalReferences.ts'; +import {JsonSchemaVisitor, type VisitorContext} from '@/schema/jsonSchemaVisitor.ts'; const settings = useSettings(); @@ -70,122 +71,86 @@ export function populateGraph( } } -export function identifyAllObjects(rootSchema: TopLevelSchema): Map { - const objectDefs = new Map(); - identifyObjects([], rootSchema, objectDefs, false, rootSchema); - - if (rootSchema.$defs) { - for (const [key, value] of Object.entries(rootSchema.$defs)) { - identifyObjects(['$defs', key], value, objectDefs, true, rootSchema); - } - } - if (rootSchema.definitions) { - for (const [key, value] of Object.entries(rootSchema.definitions)) { - identifyObjects(['definitions', key], value, objectDefs, true, rootSchema); - } - } - - return objectDefs; -} +class IdentifyObjectsVisitor extends JsonSchemaVisitor { + readonly defs = new Map(); -export function identifyObjects( - currentPath: Path, - schema: JsonSchemaType, - defs: Map, - hasUserDefinedName: boolean, - rootSchema: TopLevelSchema -) { - if (schema === true || schema === false) { - return; + constructor(private readonly rootSchema: TopLevelSchema) { + super(); } - // It can be that simple types, such as strings with enum constraint, have their own definition. - // We allow generating a node for this, so it can be referred to by other objects. - // But we do not visualize those nodes for simple types. - defs.set( - pathToString(currentPath), - generateInitialNode(currentPath, hasUserDefinedName, schema, rootSchema) - ); - - if (schema.properties) { - for (const [key, value] of Object.entries(schema.properties)) { - if (typeof value === 'object') { - const childPath = [...currentPath, 'properties', key]; - identifyObjects(childPath, value, defs, true, rootSchema); - } - } - } - if (schema.patternProperties) { - for (const [key, value] of Object.entries(schema.patternProperties)) { - if (typeof value === 'object') { - const childPath = [...currentPath, 'patternProperties', key]; - identifyObjects(childPath, value, defs, true, rootSchema); - } + private addNode(schema: JsonSchemaType, path: Path, hasUserDefinedName: boolean): void { + if (typeof schema === 'object' && schema !== null) { + this.defs.set( + pathToString(path), + generateInitialNode( + path, + hasUserDefinedName, + schema as JsonSchemaObjectType, + this.rootSchema + ) + ); } } - if (schema.items) { - if (typeof schema.items === 'object') { - const childPath = [...currentPath, 'items']; - identifyObjects(childPath, schema.items, defs, false, rootSchema); + + protected visitSchema(schema: JsonSchemaObjectType, ctx: VisitorContext): void { + if (ctx.depth === 0) { + this.addNode(schema, [] as Path, false); } } - if (schema.oneOf) { - for (const [index, value] of schema.oneOf.entries()) { - if (typeof value === 'object') { - const childPath = [...currentPath, 'oneOf', index]; - identifyObjects(childPath, value, defs, false, rootSchema); - } - } + protected visitProperty(_name: string, schema: JsonSchemaType, ctx: VisitorContext): void { + this.addNode(schema, ctx.path as Path, true); } - if (schema.anyOf) { - for (const [index, value] of schema.anyOf.entries()) { - if (typeof value === 'object') { - const childPath = [...currentPath, 'anyOf', index]; - identifyObjects(childPath, value, defs, false, rootSchema); - } - } + + protected visitPatternProperty( + _pattern: string, + schema: JsonSchemaType, + ctx: VisitorContext + ): void { + this.addNode(schema, ctx.path as Path, true); } - if (schema.allOf) { - for (const [index, value] of schema.allOf.entries()) { - if (typeof value === 'object') { - const childPath = [...currentPath, 'allOf', index]; - identifyObjects(childPath, value, defs, false, rootSchema); - } - } + + protected visitDefinition(_name: string, schema: JsonSchemaType, ctx: VisitorContext): void { + this.addNode(schema, ctx.path as Path, true); } - if (schema.if) { - if (typeof schema.if === 'object') { - identifyObjects([...currentPath, 'if'], schema.if, defs, false, rootSchema); + + protected visitCompositional( + keyword: string, + schemas: JsonSchemaType | JsonSchemaType[], + ctx: VisitorContext + ): void { + if (keyword !== 'not' && Array.isArray(schemas)) { + schemas.forEach((schema, i) => { + this.addNode(schema, [...ctx.path, keyword, i] as Path, false); + }); } } - if (schema.then) { - if (typeof schema.then === 'object') { - identifyObjects([...currentPath, 'then'], schema.then, defs, false, rootSchema); - } + + protected visitConditional(_keyword: string, schema: JsonSchemaType, ctx: VisitorContext): void { + this.addNode(schema, ctx.path as Path, false); } - if (schema.else) { - if (typeof schema.else === 'object') { - identifyObjects([...currentPath, 'else'], schema.else, defs, false, rootSchema); + + protected visitSubSchemaKeyword( + keyword: string, + schema: JsonSchemaType, + ctx: VisitorContext + ): void { + if (keyword === 'items' || keyword === 'additionalProperties') { + this.addNode(schema, ctx.path as Path, false); } } - if (schema.additionalProperties) { - if (typeof schema.additionalProperties === 'object') { - identifyObjects( - [...currentPath, 'additionalProperties'], - schema.additionalProperties, - defs, - false, - rootSchema - ); + + protected visitRef(ref: string, ctx: VisitorContext): void { + if (isExternalRef(ref)) { + this.defs.set(ref, new SchemaExternalReferenceNodeData(ref, [...ctx.path, '$ref'] as Path)); } } - if (schema.$ref && isExternalRef(schema.$ref)) { - defs.set( - schema.$ref, - new SchemaExternalReferenceNodeData(schema.$ref, [...currentPath, '$ref']) - ); - } +} + +export function identifyAllObjects(rootSchema: TopLevelSchema): Map { + const visitor = new IdentifyObjectsVisitor(rootSchema); + visitor.traverse(rootSchema); + return visitor.defs as unknown as Map; } function generateInitialNode( @@ -355,19 +320,14 @@ function generateObjectAttributesForType( function generateEnumValues(schema: JsonSchemaObjectType): string[] { if (schema.enum) { - // @ts-ignore - return schema.enum.map(value => { - if (value) { - return value.toString(); - } else if (value === null) { - return 'null'; - } else if (value === undefined) { - return 'undefined'; - } + return (schema.enum as unknown[]).map(value => { + if (value === null) return 'null'; + if (value === undefined) return 'undefined'; + return String(value); }); } - if (schema.const) { - return [schema.const.toString()]; + if (schema.const !== undefined && schema.const !== null) { + return [String(schema.const)]; } return []; } @@ -616,7 +576,17 @@ function resolveArrayItemNode( if (referenceObject) { itemObjectPath = referenceObject.absolutePath; } - return objectDefs.get(pathToString(itemObjectPath)); + const itemNode = objectDefs.get(pathToString(itemObjectPath)); + // If the resolved item is itself an array (nested arrays), recurse until we reach + // a non-array type so that e.g. array-of-array-of-Ref still produces an edge. + if ( + itemNode && + !isSchemaThatDeservesANode(itemNode.schema) && + itemNode.schema.type === 'array' + ) { + return resolveArrayItemNode(itemNode.absolutePath, itemNode.schema, objectDefs); + } + return itemNode; } } return undefined; @@ -720,7 +690,7 @@ export function generateObjectSpecialPropertyEdges( ); } if (schema.$ref) { - if (isExternalRef(schema.$ref) && isExternalRef(schema.$ref)) { + if (isExternalRef(schema.$ref)) { generateObjectSubSchemaEdge( node, {$ref: schema.$ref}, @@ -729,6 +699,18 @@ export function generateObjectSpecialPropertyEdges( objectDefs, graph ); + } else { + // Internal $ref on a node that also has other keywords (e.g. properties). + // In JSON Schema 2019-09+ this is equivalent to allOf:[{$ref}] combined with the + // sibling keywords, so we model it as an ALL_OF edge to the referenced schema. + generateObjectSubSchemaEdge( + node, + {$ref: schema.$ref}, + [...node.absolutePath, '$ref'], + EdgeType.ALL_OF, + objectDefs, + graph + ); } } } diff --git a/meta_configurator/src/schema/jsonSchemaVisitor.ts b/meta_configurator/src/schema/jsonSchemaVisitor.ts new file mode 100644 index 000000000..2449b77f3 --- /dev/null +++ b/meta_configurator/src/schema/jsonSchemaVisitor.ts @@ -0,0 +1,378 @@ +import type {JsonSchemaObjectType, JsonSchemaType} from '@/schema/jsonSchemaType'; + +export type SchemaKeywordKind = 'keyword' | 'property' | 'pattern' | 'definition'; + +export interface VisitorContext { + readonly path: readonly (string | number)[]; + readonly depth: number; + readonly parentKeyword: string | null; + readonly parentKind: SchemaKeywordKind | null; +} + +function typeName(v: unknown): string { + if (v === null) return 'null'; + if (Array.isArray(v)) return 'array'; + return typeof v; +} + +/** + * Base class for traversing a JSON Schema (drafts 4–2020-12). + * Subclass and override whichever hooks you need. All hooks default to no-ops. + * + * @param strict When true (default), malformed keyword values throw a TypeError. + * When false, invalid keywords are silently skipped. + * + * Hooks fired in traversal order per schema node: + * - visitSchema – every object schema, before keyword traversal + * - visitBooleanSchema – boolean schemas (true / false) + * - visitRef – $ref, $dynamicRef, $recursiveRef values + * - visitProperty – each named entry in `properties` + * - visitPatternProperty – each entry in `patternProperties` + * - visitSubSchemaKeyword – single-schema keywords (items, additionalProperties, …) + * - visitCompositional – allOf / anyOf / oneOf / not + * - visitConditional – if / then / else + * - visitDefinition – each entry in $defs, definitions, dependentSchemas, + * or schema-valued dependencies + */ +export abstract class JsonSchemaVisitor { + constructor(readonly strict = true) {} + + traverse(schema: JsonSchemaType): void { + this.walk(schema, {path: [], depth: 0, parentKeyword: null, parentKind: null}); + } + + protected visitSchema(_schema: JsonSchemaObjectType, _context: VisitorContext): void {} + protected visitBooleanSchema(_value: boolean, _context: VisitorContext): void {} + protected visitRef(_ref: string, _context: VisitorContext): void {} + protected visitProperty(_name: string, _schema: JsonSchemaType, _context: VisitorContext): void {} + protected visitPatternProperty( + _pattern: string, + _schema: JsonSchemaType, + _context: VisitorContext + ): void {} + protected visitSubSchemaKeyword( + _keyword: string, + _schema: JsonSchemaType, + _context: VisitorContext + ): void {} + protected visitCompositional( + _keyword: string, + _schemas: JsonSchemaType | JsonSchemaType[], + _context: VisitorContext + ): void {} + protected visitConditional( + _keyword: string, + _schema: JsonSchemaType, + _context: VisitorContext + ): void {} + protected visitDefinition( + _name: string, + _schema: JsonSchemaType, + _context: VisitorContext + ): void {} + + private invalid( + keyword: string, + expected: string, + path: readonly (string | number)[], + actual: unknown + ): false { + if (this.strict) { + throw new TypeError( + `[/${path.join('/')}] "${keyword}" must be ${expected}, got ${typeName(actual)}` + ); + } + return false; + } + + private checkStr(v: unknown, keyword: string, path: readonly (string | number)[]): v is string { + return typeof v === 'string' || this.invalid(keyword, 'a string', path, v); + } + + private checkNum(v: unknown, keyword: string, path: readonly (string | number)[]): v is number { + return typeof v === 'number' || this.invalid(keyword, 'a number', path, v); + } + + private checkBool(v: unknown, keyword: string, path: readonly (string | number)[]): v is boolean { + return typeof v === 'boolean' || this.invalid(keyword, 'a boolean', path, v); + } + + private checkArr( + v: unknown, + keyword: string, + path: readonly (string | number)[] + ): v is unknown[] { + return Array.isArray(v) || this.invalid(keyword, 'an array', path, v); + } + + private checkObj( + v: unknown, + keyword: string, + path: readonly (string | number)[] + ): v is Record { + return ( + (typeof v === 'object' && v !== null && !Array.isArray(v)) || + this.invalid(keyword, 'an object', path, v) + ); + } + + private checkSchema( + v: unknown, + keyword: string, + path: readonly (string | number)[] + ): v is JsonSchemaType { + return ( + typeof v === 'boolean' || + (typeof v === 'object' && v !== null && !Array.isArray(v)) || + this.invalid(keyword, 'a schema (boolean or object)', path, v) + ); + } + + private checkStrOrArr( + v: unknown, + keyword: string, + path: readonly (string | number)[] + ): v is string | string[] { + return ( + typeof v === 'string' || + Array.isArray(v) || + this.invalid(keyword, 'a string or array', path, v) + ); + } + + private checkNumOrBool( + v: unknown, + keyword: string, + path: readonly (string | number)[] + ): v is number | boolean { + return ( + typeof v === 'number' || + typeof v === 'boolean' || + this.invalid(keyword, 'a number or boolean', path, v) + ); + } + + private walk(value: unknown, context: VisitorContext): void { + if (typeof value === 'boolean') { + this.visitBooleanSchema(value, context); + return; + } + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + this.invalid('(schema)', 'a boolean or object', context.path, value); + return; + } + const schema = value as Record; + this.visitSchema(schema as JsonSchemaObjectType, context); + this.walkKeywords(schema, context); + } + + private child( + context: VisitorContext, + keyword: string, + segments: (string | number)[], + kind: SchemaKeywordKind = 'keyword' + ): VisitorContext { + return { + path: [...context.path, ...segments], + depth: context.depth + 1, + parentKeyword: keyword, + parentKind: kind, + }; + } + + private walkKeywords(schema: Record, context: VisitorContext): void { + for (const keyword of ['$ref', '$dynamicRef', '$recursiveRef'] as const) { + if (keyword in schema && this.checkStr(schema[keyword], keyword, context.path)) { + this.visitRef(schema[keyword], context); + } + } + + // scalar keyword validation — no hooks, values accessible via visitSchema + if ('$schema' in schema) this.checkStr(schema.$schema, '$schema', context.path); + if ('$id' in schema) this.checkStr(schema.$id, '$id', context.path); + if ('id' in schema) this.checkStr(schema.id, 'id', context.path); + if ('$anchor' in schema) this.checkStr(schema.$anchor, '$anchor', context.path); + if ('$dynamicAnchor' in schema) + this.checkStr(schema.$dynamicAnchor, '$dynamicAnchor', context.path); + if ('$recursiveAnchor' in schema) + this.checkStr(schema.$recursiveAnchor, '$recursiveAnchor', context.path); + if ('$comment' in schema) this.checkStr(schema.$comment, '$comment', context.path); + if ('title' in schema) this.checkStr(schema.title, 'title', context.path); + if ('description' in schema) this.checkStr(schema.description, 'description', context.path); + if ('examples' in schema) this.checkArr(schema.examples, 'examples', context.path); + if ('deprecated' in schema) this.checkBool(schema.deprecated, 'deprecated', context.path); + if ('readOnly' in schema) this.checkBool(schema.readOnly, 'readOnly', context.path); + if ('writeOnly' in schema) this.checkBool(schema.writeOnly, 'writeOnly', context.path); + if ('type' in schema) this.checkStrOrArr(schema.type, 'type', context.path); + if ('enum' in schema) this.checkArr(schema.enum, 'enum', context.path); + if ('format' in schema) this.checkStr(schema.format, 'format', context.path); + if ('pattern' in schema) this.checkStr(schema.pattern, 'pattern', context.path); + if ('contentMediaType' in schema) + this.checkStr(schema.contentMediaType, 'contentMediaType', context.path); + if ('contentEncoding' in schema) + this.checkStr(schema.contentEncoding, 'contentEncoding', context.path); + if ('required' in schema) this.checkArr(schema.required, 'required', context.path); + if ('dependentRequired' in schema) + this.checkObj(schema.dependentRequired, 'dependentRequired', context.path); + if ('minLength' in schema) this.checkNum(schema.minLength, 'minLength', context.path); + if ('maxLength' in schema) this.checkNum(schema.maxLength, 'maxLength', context.path); + if ('minimum' in schema) this.checkNum(schema.minimum, 'minimum', context.path); + if ('maximum' in schema) this.checkNum(schema.maximum, 'maximum', context.path); + if ('exclusiveMinimum' in schema) + this.checkNumOrBool(schema.exclusiveMinimum, 'exclusiveMinimum', context.path); + if ('exclusiveMaximum' in schema) + this.checkNumOrBool(schema.exclusiveMaximum, 'exclusiveMaximum', context.path); + if ('multipleOf' in schema) this.checkNum(schema.multipleOf, 'multipleOf', context.path); + if ('minItems' in schema) this.checkNum(schema.minItems, 'minItems', context.path); + if ('maxItems' in schema) this.checkNum(schema.maxItems, 'maxItems', context.path); + if ('uniqueItems' in schema) this.checkBool(schema.uniqueItems, 'uniqueItems', context.path); + if ('minContains' in schema) this.checkNum(schema.minContains, 'minContains', context.path); + if ('maxContains' in schema) this.checkNum(schema.maxContains, 'maxContains', context.path); + if ('minProperties' in schema) + this.checkNum(schema.minProperties, 'minProperties', context.path); + if ('maxProperties' in schema) + this.checkNum(schema.maxProperties, 'maxProperties', context.path); + + if ('properties' in schema && this.checkObj(schema.properties, 'properties', context.path)) { + for (const [name, propSchema] of Object.entries( + schema.properties as Record + )) { + if (this.checkSchema(propSchema, `properties["${name}"]`, context.path)) { + const propCtx = this.child(context, name, ['properties', name], 'property'); + this.visitProperty(name, propSchema, propCtx); + this.walk(propSchema, propCtx); + } + } + } + + if ( + 'patternProperties' in schema && + this.checkObj(schema.patternProperties, 'patternProperties', context.path) + ) { + for (const [pattern, patSchema] of Object.entries( + schema.patternProperties as Record + )) { + if (this.checkSchema(patSchema, `patternProperties["${pattern}"]`, context.path)) { + const patCtx = this.child(context, pattern, ['patternProperties', pattern], 'pattern'); + this.visitPatternProperty(pattern, patSchema, patCtx); + this.walk(patSchema, patCtx); + } + } + } + + for (const keyword of [ + 'additionalProperties', + 'unevaluatedProperties', + 'propertyNames', + 'additionalItems', + 'unevaluatedItems', + 'contains', + 'contentSchema', + ]) { + if (keyword in schema && this.checkSchema(schema[keyword], keyword, context.path)) { + const keywordContext = this.child(context, keyword, [keyword]); + this.visitSubSchemaKeyword(keyword, schema[keyword] as JsonSchemaType, keywordContext); + this.walk(schema[keyword], keywordContext); + } + } + + if ('items' in schema) { + if (Array.isArray(schema.items)) { + schema.items.forEach((item, i) => { + if (this.checkSchema(item, `items[${i}]`, context.path)) { + const itemCtx = this.child(context, 'items', ['items', i]); + this.visitSubSchemaKeyword('items', item, itemCtx); + this.walk(item, itemCtx); + } + }); + } else if (this.checkSchema(schema.items, 'items', context.path)) { + const itemCtx = this.child(context, 'items', ['items']); + this.visitSubSchemaKeyword('items', schema.items, itemCtx); + this.walk(schema.items, itemCtx); + } + } + + if ('prefixItems' in schema && this.checkArr(schema.prefixItems, 'prefixItems', context.path)) { + schema.prefixItems.forEach((item, i) => { + if (this.checkSchema(item, `prefixItems[${i}]`, context.path)) { + const itemCtx = this.child(context, 'prefixItems', ['prefixItems', i]); + this.visitSubSchemaKeyword('prefixItems', item, itemCtx); + this.walk(item, itemCtx); + } + }); + } + + for (const keyword of ['allOf', 'anyOf', 'oneOf']) { + if (keyword in schema && this.checkArr(schema[keyword], keyword, context.path)) { + const schemas = schema[keyword] as unknown[]; + this.visitCompositional(keyword, schemas as JsonSchemaType[], context); + schemas.forEach((s, i) => { + if (this.checkSchema(s, `${keyword}[${i}]`, context.path)) { + this.walk(s, this.child(context, keyword, [keyword, i])); + } + }); + } + } + + if ('not' in schema && this.checkSchema(schema.not, 'not', context.path)) { + this.visitCompositional('not', schema.not, context); + this.walk(schema.not, this.child(context, 'not', ['not'])); + } + + for (const keyword of ['if', 'then', 'else']) { + if (keyword in schema && this.checkSchema(schema[keyword], keyword, context.path)) { + const keywordContext = this.child(context, keyword, [keyword]); + this.visitConditional(keyword, schema[keyword] as JsonSchemaType, keywordContext); + this.walk(schema[keyword], keywordContext); + } + } + + for (const defsKeyword of ['$defs', 'definitions']) { + if (defsKeyword in schema && this.checkObj(schema[defsKeyword], defsKeyword, context.path)) { + for (const [name, defSchema] of Object.entries( + schema[defsKeyword] as Record + )) { + if (this.checkSchema(defSchema, `${defsKeyword}["${name}"]`, context.path)) { + const definitionContext = this.child(context, name, [defsKeyword, name], 'definition'); + this.visitDefinition(name, defSchema, definitionContext); + this.walk(defSchema, definitionContext); + } + } + } + } + + if ( + 'dependentSchemas' in schema && + this.checkObj(schema.dependentSchemas, 'dependentSchemas', context.path) + ) { + for (const [name, depSchema] of Object.entries( + schema.dependentSchemas as Record + )) { + if (this.checkSchema(depSchema, `dependentSchemas["${name}"]`, context.path)) { + const dependentContext = this.child( + context, + name, + ['dependentSchemas', name], + 'definition' + ); + this.visitDefinition(name, depSchema, dependentContext); + this.walk(depSchema, dependentContext); + } + } + } + + if ( + 'dependencies' in schema && + this.checkObj(schema.dependencies, 'dependencies', context.path) + ) { + for (const [key, dep] of Object.entries(schema.dependencies as Record)) { + if (Array.isArray(dep)) continue; + if (this.checkSchema(dep, `dependencies["${key}"]`, context.path)) { + const dependenciesContext = this.child(context, key, ['dependencies', key], 'definition'); + this.visitDefinition(key, dep, dependenciesContext); + this.walk(dep, dependenciesContext); + } + } + } + } +} diff --git a/meta_configurator/src/schema/metaSchemaBuilder.ts b/meta_configurator/src/schema/metaSchemaBuilder.ts index 6d50401ac..e72b1be0a 100644 --- a/meta_configurator/src/schema/metaSchemaBuilder.ts +++ b/meta_configurator/src/schema/metaSchemaBuilder.ts @@ -4,7 +4,6 @@ import {META_SCHEMA_SIMPLIFIED} from '@/packaged-schemas/metaSchemaSimplified'; import {updateSettingsWithDefaults} from '@/settings/settingsUpdater'; import {SETTINGS_DATA_DEFAULT} from '@/settings/defaultSettingsData'; -// TODO: auto-suggest or enable features if user loads a schema that requires them export function buildMetaSchema(metaSchemaSettings: SettingsInterfaceMetaSchema): TopLevelSchema { let metaSchema = structuredClone(META_SCHEMA_SIMPLIFIED); diff --git a/meta_configurator/src/utility/ai/aiEndpoint.ts b/meta_configurator/src/utility/ai/aiEndpoint.ts index c3e58f210..3b182d21b 100644 --- a/meta_configurator/src/utility/ai/aiEndpoint.ts +++ b/meta_configurator/src/utility/ai/aiEndpoint.ts @@ -25,7 +25,7 @@ export const queryOpenAI = async ( } try { - console.log('Querying OpenAI with messages: ', ...messages); + console.debug('Querying AI endpoint with messages: ', ...messages); const response = await axios.post( endpoint, { @@ -42,7 +42,7 @@ export const queryOpenAI = async ( } ); const resultSchema: string = response.data.choices[0].message.content; - console.log('Result schema from AI prompt:', resultSchema, 'based on messages:', messages); + console.debug('Result schema from AI prompt:', resultSchema, 'based on messages:', messages); return resultSchema; } catch (error) { console.error('Error querying OpenAI:', error); diff --git a/meta_configurator/src/utility/ai/apiKey.ts b/meta_configurator/src/utility/ai/apiKey.ts index 646cb1567..157b29a7b 100644 --- a/meta_configurator/src/utility/ai/apiKey.ts +++ b/meta_configurator/src/utility/ai/apiKey.ts @@ -1,47 +1,57 @@ import {ref, type Ref, watch} from 'vue'; -const initialized: Ref = ref(false); -const apiKey: Ref = ref(''); -const isPersistKey: Ref = ref(true); +/** + * API key handling for user-provided credentials. + * + * The key lives in memory by default and is lost on page refresh. If the user + * opts in, it is additionally saved to sessionStorage so it survives a refresh + * within the same tab but is cleared when the tab or browser is closed. + * + * localStorage is intentionally not used: both sessionStorage and localStorage + * are readable by same-origin scripts, so neither offers protection against + * active JavaScript on the page. The only meaningful difference is persistence + * lifetime. Limiting to sessionStorage reduces the window of exposure. + */ -export function iniApiKey() { - if (initialized.value) { - return; - } - const storedApiKey = localStorage.getItem('openai_api_key'); - if (storedApiKey) { - apiKey.value = storedApiKey; - } - const storedPersistKey = localStorage.getItem('openai_persist_key'); - if (storedPersistKey) { - isPersistKey.value = storedPersistKey === 'true'; - } - initialized.value = true; -} +const STORAGE_KEY = 'mc_api_key'; -export function getApiKeyRef(): Ref { - if (!initialized.value) { - iniApiKey(); - } - return apiKey; -} +const apiKey: Ref = ref(''); +const rememberInTab: Ref = ref(false); -export function getIsPersistKeyRef(): Ref { - if (!initialized.value) { - iniApiKey(); +export function initApiKey(): void { + const stored = sessionStorage.getItem(STORAGE_KEY); + if (stored) { + apiKey.value = stored; + rememberInTab.value = true; } - return isPersistKey; } watch(apiKey, newValue => { - if (isPersistKey.value) { - localStorage.setItem('openai_api_key', newValue); + if (!rememberInTab.value) return; + if (newValue) { + sessionStorage.setItem(STORAGE_KEY, newValue); + } else { + sessionStorage.removeItem(STORAGE_KEY); } }); -watch(isPersistKey, newValue => { - localStorage.setItem('openai_persist_key', newValue.toString()); - if (!newValue) { - localStorage.removeItem('openai_api_key'); +watch(rememberInTab, newValue => { + if (newValue && apiKey.value) { + sessionStorage.setItem(STORAGE_KEY, apiKey.value); + } else { + sessionStorage.removeItem(STORAGE_KEY); } }); + +/** Returns the current API key value. Valid synchronously after initApiKey() is called. */ +export function getApiKey(): string { + return apiKey.value; +} + +export function getApiKeyRef(): Ref { + return apiKey; +} + +export function getRememberInTabRef(): Ref { + return rememberInTab; +} diff --git a/meta_configurator/src/utility/deleteUtils.ts b/meta_configurator/src/utility/deleteUtils.ts index bbfcd21bd..238c120ff 100644 --- a/meta_configurator/src/utility/deleteUtils.ts +++ b/meta_configurator/src/utility/deleteUtils.ts @@ -1,6 +1,7 @@ import type {ManagedData} from '@/data/managedData'; import type {Path} from '@/utility/path'; import {getParentElementRequiredPropsPath} from '@/utility/pathUtils'; +import {removeFromRequiredArray} from '@/utility/requiredUtils'; export function deleteSchemaElement(schema: ManagedData, absolutePath: Path) { schema.removeDataAt(absolutePath); @@ -12,12 +13,9 @@ export function deleteSchemaElement(schema: ManagedData, absolutePath: Path) { ); if (parentRequiredPropsPath) { const requiredProps = schema.dataAt(parentRequiredPropsPath); - const schemaElementName = absolutePath[absolutePath.length - 1]; - const requiredIndex = requiredProps.indexOf(schemaElementName); - if (requiredIndex !== -1) { - const updatedRequiredProps = requiredProps.filter( - (_: string, index: number) => index !== requiredIndex - ); + const schemaElementName = absolutePath[absolutePath.length - 1] as string; + const updatedRequiredProps = removeFromRequiredArray(requiredProps, schemaElementName); + if (updatedRequiredProps !== requiredProps) { schema.setDataAt(parentRequiredPropsPath, updatedRequiredProps); } } diff --git a/meta_configurator/src/utility/documentation/__tests__/samples/arrays.expected.md b/meta_configurator/src/utility/documentation/__tests__/samples/arrays.expected.md index 9acd6d63b..c6563766e 100644 --- a/meta_configurator/src/utility/documentation/__tests__/samples/arrays.expected.md +++ b/meta_configurator/src/utility/documentation/__tests__/samples/arrays.expected.md @@ -6,15 +6,15 @@ - [nestedArraysAndObjectsProperty entry](#%2Fproperties%2FnestedArraysAndObjectsProperty%2Fitems) --- -### [My Schema](#root) +### [My Schema](#root) #### Properties | Name | Type | Required | Description | |------|------|------|------| -| numberArrayProperty | number\[\] | false | \- | -| objectArrayProperty | [objectArrayProperty entry\[\]](#%2Fproperties%2FobjectArrayProperty%2Fitems) | false | \- | -| nestedArraysProperty | nestedArraysProperty entry\[\] | false | \- | -| nestedArraysAndObjectsProperty | [nestedArraysAndObjectsProperty entry\[\]](#%2Fproperties%2FnestedArraysAndObjectsProperty%2Fitems) | false | \- | +| numberArrayProperty | number\[\] | false | \- | +| objectArrayProperty | [objectArrayProperty entry\[\]](#%2Fproperties%2FobjectArrayProperty%2Fitems) | false | \- | +| nestedArraysProperty | string\[\] | false | \- | +| nestedArraysAndObjectsProperty | [nestedArraysAndObjectsProperty entry\[\]](#%2Fproperties%2FnestedArraysAndObjectsProperty%2Fitems) | false | \- | #### Example @@ -45,12 +45,12 @@ } ``` --- -### [objectArrayProperty entry](#%2Fproperties%2FobjectArrayProperty%2Fitems) +### [objectArrayProperty entry](#%2Fproperties%2FobjectArrayProperty%2Fitems) #### Properties | Name | Type | Required | Description | |------|------|------|------| -| booleanProperty | boolean | false | \- | +| booleanProperty | boolean | false | \- | #### Example @@ -60,12 +60,12 @@ } ``` --- -### [nestedArraysAndObjectsProperty entry](#%2Fproperties%2FnestedArraysAndObjectsProperty%2Fitems) +### [nestedArraysAndObjectsProperty entry](#%2Fproperties%2FnestedArraysAndObjectsProperty%2Fitems) #### Properties | Name | Type | Required | Description | |------|------|------|------| -| arrayProp | string\[\] | false | \- | +| arrayProp | string\[\] | false | \- | #### Example diff --git a/meta_configurator/src/utility/documentation/__tests__/samples/featureTesting.expected.md b/meta_configurator/src/utility/documentation/__tests__/samples/featureTesting.expected.md index 281b71586..b9ef2f92d 100644 --- a/meta_configurator/src/utility/documentation/__tests__/samples/featureTesting.expected.md +++ b/meta_configurator/src/utility/documentation/__tests__/samples/featureTesting.expected.md @@ -337,4 +337,4 @@ --- ### [No Partner](#%2Fproperties%2Fpartner%2FoneOf%2F0) #### Enumeration Values -- `undefined` +- `false` diff --git a/meta_configurator/src/utility/documentation/__tests__/samples/references.expected.md b/meta_configurator/src/utility/documentation/__tests__/samples/references.expected.md index 9c7743b44..5b4e91fe4 100644 --- a/meta_configurator/src/utility/documentation/__tests__/samples/references.expected.md +++ b/meta_configurator/src/utility/documentation/__tests__/samples/references.expected.md @@ -5,12 +5,18 @@ - [person](#%2F%24defs%2Fperson) - [person](#%2F%24defs%2Fperson) - [person1](#%2F%24defs%2Fperson) + - [person](#%2F%24defs%2Fperson) + - [person](#%2F%24defs%2Fperson) - [person](#%2F%24defs%2Fperson) - [person](#%2F%24defs%2Fperson) - [person2](#%2F%24defs%2Fperson) + - [person](#%2F%24defs%2Fperson) + - [person](#%2F%24defs%2Fperson) - [person](#%2F%24defs%2Fperson) - [person](#%2F%24defs%2Fperson) - [person3](#%2F%24defs%2Fperson) + - [person](#%2F%24defs%2Fperson) + - [person](#%2F%24defs%2Fperson) - [person](#%2F%24defs%2Fperson) - [person](#%2F%24defs%2Fperson) - [person](#%2F%24defs%2Fperson) diff --git a/meta_configurator/src/utility/documentation/__tests__/samples/titles.expected.md b/meta_configurator/src/utility/documentation/__tests__/samples/titles.expected.md index 9b7e177d0..b62596541 100644 --- a/meta_configurator/src/utility/documentation/__tests__/samples/titles.expected.md +++ b/meta_configurator/src/utility/documentation/__tests__/samples/titles.expected.md @@ -6,9 +6,13 @@ - [PersonObject](#%2F%24defs%2Fperson) - [PersonObject](#%2F%24defs%2Fperson) - [person1](#%2F%24defs%2Fperson) + - [PersonObject](#%2F%24defs%2Fperson) + - [PersonObject](#%2F%24defs%2Fperson) - [PersonObject](#%2F%24defs%2Fperson) - [PersonObject](#%2F%24defs%2Fperson) - [person2](#%2F%24defs%2Fperson) + - [PersonObject](#%2F%24defs%2Fperson) + - [PersonObject](#%2F%24defs%2Fperson) - [PersonObject](#%2F%24defs%2Fperson) - [PersonObject](#%2F%24defs%2Fperson) - [PersonObject](#%2F%24defs%2Fperson) diff --git a/meta_configurator/src/utility/documentation/__tests__/schemaToMarkdown.test.ts b/meta_configurator/src/utility/documentation/__tests__/schemaToMarkdown.test.ts index 15aa818b9..f655ad0db 100644 --- a/meta_configurator/src/utility/documentation/__tests__/schemaToMarkdown.test.ts +++ b/meta_configurator/src/utility/documentation/__tests__/schemaToMarkdown.test.ts @@ -74,6 +74,10 @@ describe('schemaToMarkdown samples coverage', async () => { ); const expected = cleanMarkdownContent(expectedMd.trimEnd()); // ignore trailing newline diffs + // print source schema path for easier debugging + if (actualMd !== expected) { + console.error(`Schema path: ${c.schemaPath}`); + } expect(actualMd).toBe(expected); }); } diff --git a/meta_configurator/src/utility/renameUtils.ts b/meta_configurator/src/utility/renameUtils.ts index e75136d34..10bc1aa0f 100644 --- a/meta_configurator/src/utility/renameUtils.ts +++ b/meta_configurator/src/utility/renameUtils.ts @@ -2,6 +2,7 @@ import type {Path} from '@/utility/path'; import {dataAt} from '@/utility/resolveDataAtPath'; import type {JsonSchemaWrapper} from '@/schema/jsonSchemaWrapper'; import {getParentElementRequiredPropsPath, pathToJsonPointer} from '@/utility/pathUtils'; +import {removeFromRequiredArray} from '@/utility/requiredUtils'; import {SessionMode} from '@/store/sessionMode'; export function replacePropertyNameUtils( @@ -61,11 +62,8 @@ export function updateParentRequiredPropsValue( ); if (parentRequiredPropsPath) { const requiredProps = dataAt(parentRequiredPropsPath, schemaData) ?? []; - const requiredIndex = requiredProps.indexOf(oldPropertyName); - if (requiredIndex !== -1) { - const updatedRequiredProps = requiredProps.filter( - (_: string, index: number) => index !== requiredIndex - ); + const updatedRequiredProps = removeFromRequiredArray(requiredProps, oldPropertyName); + if (updatedRequiredProps !== requiredProps) { updatedRequiredProps.push(newPropertyName); updateDataFct(parentRequiredPropsPath, updatedRequiredProps); } diff --git a/meta_configurator/src/utility/requiredUtils.ts b/meta_configurator/src/utility/requiredUtils.ts new file mode 100644 index 000000000..02748209b --- /dev/null +++ b/meta_configurator/src/utility/requiredUtils.ts @@ -0,0 +1,9 @@ +/** + * Returns a new array with the given name removed from the required props list. + * Returns the original array unchanged if the name is not present. + */ +export function removeFromRequiredArray(requiredProps: string[], nameToRemove: string): string[] { + const index = requiredProps.indexOf(nameToRemove); + if (index === -1) return requiredProps; + return requiredProps.filter((_, i) => i !== index); +} diff --git a/meta_configurator/vite.config.js b/meta_configurator/vite.config.js index 4386ebe6d..9a1159273 100644 --- a/meta_configurator/vite.config.js +++ b/meta_configurator/vite.config.js @@ -11,12 +11,14 @@ const pkg = require('./package.json'); // Use process.env.USE_BASE_PATH to determine if base path should be included const useMetaConfiguratorBasePath = process.env.USE_META_CONFIGURATOR_BASE_PATH === 'true'; +const isExperimental = process.env.EXPERIMENTAL !== 'false'; // https://vitejs.dev/config/ export default defineConfig({ base: useMetaConfiguratorBasePath ? '/meta-configurator/' : '/', define: { __APP_VERSION__: JSON.stringify(pkg.version), + __APP_EXPERIMENTAL__: JSON.stringify(isExperimental), }, plugins: [vue(), vueJsx(),