|
| 1 | +export function trimDataToMaxSize( |
| 2 | + data: any, |
| 3 | + maxSizeInKiB: number = 64, |
| 4 | + minObjectPropertyCountToPreserve: number = 16 |
| 5 | +): any { |
| 6 | + let n = 64; |
| 7 | + |
| 8 | + // cut data to n entries and check size. If it is not yet reached, divide n by 2 and repeat. |
| 9 | + // minimum n is 2 |
| 10 | + while (true) { |
| 11 | + const dataTrimmedArrays = trimDataToNEntriesPerArray(data, n); |
| 12 | + // we trim object properties to n*8, because even in the minimum case we want to preserve at least 16 properties (or more, if defined in the parameter minObjectPropertyCountToPreserve). |
| 13 | + // properties normally should not be cut at all, except the schema uses patternProperties or additionalProperties and the user has hundreds or millions of same-looking objects |
| 14 | + // therefore, trim object properties very conservatively only, but do it if necessary. |
| 15 | + const dataTrimmedBoth = trimDataToNPropertiesPerObject( |
| 16 | + dataTrimmedArrays, |
| 17 | + Math.max(n * 8, minObjectPropertyCountToPreserve) |
| 18 | + ); |
| 19 | + const sizeInBytes = new TextEncoder().encode(JSON.stringify(dataTrimmedBoth)).length; |
| 20 | + const sizeInKiB = sizeInBytes / 1024; // convert to KiB |
| 21 | + if (sizeInKiB <= maxSizeInKiB || n <= 2) { |
| 22 | + return dataTrimmedBoth; // return the cut data if size is within limit or n is too small |
| 23 | + } |
| 24 | + |
| 25 | + n = Math.floor(n / 2); // reduce n by half |
| 26 | + } |
| 27 | +} |
| 28 | + |
| 29 | +function trimDataToNEntriesPerArray(data: any, n: number): any { |
| 30 | + // data will be a json object or array with an arbitrary hierarchy and anywhere could be arrays |
| 31 | + // we want to cut each array to have only n entries |
| 32 | + |
| 33 | + // check if data is an array. Even then, children could be objects or arrays. Apply same algorithm recursively on each array item |
| 34 | + if (Array.isArray(data)) { |
| 35 | + const newArray = []; |
| 36 | + let i = 0; |
| 37 | + for (const item of data) { |
| 38 | + // if the array has more than n entries, cut it to n entries |
| 39 | + if (i < n) { |
| 40 | + i++; |
| 41 | + } else { |
| 42 | + break; |
| 43 | + } |
| 44 | + newArray.push(trimDataToNEntriesPerArray(item, n)); |
| 45 | + } |
| 46 | + return newArray; |
| 47 | + } |
| 48 | + |
| 49 | + // if data is an object, we need to traverse the object and cut each array to have only 3 entries |
| 50 | + if (typeof data === 'object' && data !== null) { |
| 51 | + const newObject: any = {}; |
| 52 | + for (const key in data) { |
| 53 | + newObject[key] = trimDataToNEntriesPerArray(data[key], n); |
| 54 | + } |
| 55 | + return newObject; |
| 56 | + } |
| 57 | + // if data is not an object or array, return it as is |
| 58 | + return data; |
| 59 | +} |
| 60 | + |
| 61 | +function trimDataToNPropertiesPerObject(data: any, n: number): any { |
| 62 | + // data will be a json object or array with an arbitrary hierarchy and anywhere could be objects |
| 63 | + // we want to cut each object to have only n properties |
| 64 | + |
| 65 | + // check if data is an array. Even then, children could be objects or arrays. Apply same algorithm recursively on each array item |
| 66 | + if (Array.isArray(data)) { |
| 67 | + return data.map(item => trimDataToNPropertiesPerObject(item, n)); |
| 68 | + } |
| 69 | + |
| 70 | + // if data is an object, we need to traverse the object and cut each object to have only n properties |
| 71 | + if (typeof data === 'object' && data !== null) { |
| 72 | + const newObject: any = {}; |
| 73 | + const keys = Object.keys(data); |
| 74 | + for (let i = 0; i < Math.min(n, keys.length); i++) { |
| 75 | + const key = keys[i]; |
| 76 | + newObject[key] = trimDataToNPropertiesPerObject(data[key], n); |
| 77 | + } |
| 78 | + return newObject; |
| 79 | + } |
| 80 | + // if data is not an object or array, return it as is |
| 81 | + return data; |
| 82 | +} |
0 commit comments