Fix TcfCaV1 (Canada) OptimizedRange + base64 encoding, with backwards-compatible decoding

modules/cmpapi/src/encoder/datatype/EncodableArrayOfOptimizedFibonacciRanges.ts

@@ -0,0 +1,101 @@
+import { DecodingError } from "../error/DecodingError.js";
+import { EncodingError } from "../error/EncodingError.js";
+import { StringUtil } from "../util/StringUtil.js";
+import { AbstractEncodableBitStringDataType } from "./AbstractEncodableBitStringDataType.js";
+import { EncodableOptimizedFibonacciRange } from "./EncodableOptimizedFibonacciRange.js";
+import { RangeEntry } from "./RangeEntry.js";
+import { SubstringError } from "./SubstringError.js";
+import { FixedIntegerEncoder } from "./encoder/FixedIntegerEncoder.js";
+
+/**
+ * Encodes an N-ArrayOfRanges(X,Y) field where each record's ids are encoded as an OptimizedRange
+ * (Fibonacci coded), per the GPP Consent String Specification. This is the counterpart to
+ * EncodableArrayOfFixedIntegerRanges, which encodes ids using fixed-integer ranges for downward
+ * compatibility (e.g. TCF EU).
+ */
+export class EncodableArrayOfOptimizedFibonacciRanges extends AbstractEncodableBitStringDataType<RangeEntry[]> {
+  private keyBitStringLength: number;
+  private typeBitStringLength: number;
+
+  constructor(
+    keyBitStringLength: number,
+    typeBitStringLength: number,
+    value?: RangeEntry[],
+    hardFailIfMissing: boolean = true
+  ) {
+    super(hardFailIfMissing);
+    this.keyBitStringLength = keyBitStringLength;
+    this.typeBitStringLength = typeBitStringLength;
+    this.setValue(value);
+  }
+
+  public encode(): string {
+    try {
+      let entries: RangeEntry[] = this.value;
+      let sb = "";
+      sb += FixedIntegerEncoder.encode(entries.length, 12);
+      for (let i = 0; i < entries.length; i++) {
+        let entry: RangeEntry = entries[i];
+        sb += FixedIntegerEncoder.encode(entry.getKey(), this.keyBitStringLength);
+        sb += FixedIntegerEncoder.encode(entry.getType(), this.typeBitStringLength);
+        sb += new EncodableOptimizedFibonacciRange(entry.getIds()).encode();
+      }
+      return sb;
+    } catch (e) {
+      throw new EncodingError(e);
+    }
+  }
+
+  public decode(bitString: string) {
+    try {
+      let entries: RangeEntry[] = [];
+      let size = FixedIntegerEncoder.decode(StringUtil.substring(bitString, 0, 12));
+      let index = 12;
+      for (let i = 0; i < size; i++) {
+        let key = FixedIntegerEncoder.decode(StringUtil.substring(bitString, index, index + this.keyBitStringLength));
+        index += this.keyBitStringLength;
+
+        let type = FixedIntegerEncoder.decode(StringUtil.substring(bitString, index, index + this.typeBitStringLength));
+        index += this.typeBitStringLength;
+
+        let range = new EncodableOptimizedFibonacciRange([]);
+        let substring = range.substring(bitString, index);
+        range.decode(substring);
+        let ids = range.getValue();
+        index += substring.length;
+
+        entries.push(new RangeEntry(key, type, ids));
+      }
+      this.value = entries;
+    } catch (e) {
+      throw new DecodingError(e);
+    }
+  }
+
+  public substring(bitString: string, fromIndex: number): string {
+    try {
+      let sb = "";
+      sb += StringUtil.substring(bitString, fromIndex, fromIndex + 12);
+
+      let size = FixedIntegerEncoder.decode(sb.toString());
+
+      let index = fromIndex + sb.length;
+      for (let i = 0; i < size; i++) {
+        let keySubstring = StringUtil.substring(bitString, index, index + this.keyBitStringLength);
+        index += keySubstring.length;
+        sb += keySubstring;
+
+        let typeSubstring = StringUtil.substring(bitString, index, index + this.typeBitStringLength);
+        index += typeSubstring.length;
+        sb += typeSubstring;
+
+        let rangeSubstring = new EncodableOptimizedFibonacciRange([]).substring(bitString, index);
+        index += rangeSubstring.length;
+        sb += rangeSubstring;
+      }
+      return sb;
+    } catch (e) {
+      throw new SubstringError(e);
+    }
+  }
+}

modules/cmpapi/src/encoder/datatype/index.ts

@@ -3,6 +3,7 @@ export * from "./validate/index.js";
 export * from "./AbstractEncodableBitStringDataType.js";
 export * from "./DataType.js";
 export * from "./EncodableArrayOfFixedIntegerRanges.js";
+export * from "./EncodableArrayOfOptimizedFibonacciRanges.js";
 export * from "./EncodableBoolean.js";
 export * from "./EncodableDataType.js";
 export * from "./EncodableDatetime.js";

modules/cmpapi/src/encoder/segment/TcfCaV1CoreSegment.ts

@@ -2,11 +2,13 @@ import { AbstractBase64UrlEncoder } from "../base64/AbstractBase64UrlEncoder.js"
 import { CompressedBase64UrlEncoder } from "../base64/CompressedBase64UrlEncoder.js";
 import { BitStringEncoder } from "../bitstring/BitStringEncoder.js";
 import { EncodableArrayOfFixedIntegerRanges } from "../datatype/EncodableArrayOfFixedIntegerRanges.js";
+import { EncodableArrayOfOptimizedFibonacciRanges } from "../datatype/EncodableArrayOfOptimizedFibonacciRanges.js";
 import { EncodableBoolean } from "../datatype/EncodableBoolean.js";
 import { EncodableDatetime } from "../datatype/EncodableDatetime.js";
 import { EncodableFixedBitfield } from "../datatype/EncodableFixedBitfield.js";
 import { EncodableFixedInteger } from "../datatype/EncodableFixedInteger.js";
 import { EncodableFixedString } from "../datatype/EncodableFixedString.js";
+import { EncodableOptimizedFibonacciRange } from "../datatype/EncodableOptimizedFibonacciRange.js";
 import { EncodableOptimizedFixedRange } from "../datatype/EncodableOptimizedFixedRange.js";
 import { DecodingError } from "../error/DecodingError.js";
 import { EncodableBitStringFields } from "../field/EncodableBitStringFields.js";
@@ -33,6 +35,16 @@ export class TcfCaV1CoreSegment extends AbstractLazilyEncodableSegment<Encodable
 
   // overriden
   protected initializeFields(): EncodableBitStringFields {
+    return this.buildFields(false);
+  }
+
+  /**
+   * Builds the core field set. When legacy is true the OptimizedRange / N-ArrayOfRanges fields use
+   * the pre-fix fixed-integer encoders; otherwise they use the spec-compliant Fibonacci encoders.
+   * The legacy field set is only used to decode strings produced by the older encoder (see
+   * decodeSegment).
+   */
+  private buildFields(legacy: boolean): EncodableBitStringFields {
     let date = new Date();
 
     let fields: EncodableBitStringFields = new EncodableBitStringFields();
@@ -108,9 +120,18 @@ export class TcfCaV1CoreSegment extends AbstractLazilyEncodableSegment<Encodable
         false,
       ])
     );
-    fields.put(TcfCaV1Field.VENDOR_EXPRESS_CONSENT.toString(), new EncodableOptimizedFixedRange([]));
-    fields.put(TcfCaV1Field.VENDOR_IMPLIED_CONSENT.toString(), new EncodableOptimizedFixedRange([]));
-    fields.put(TcfCaV1Field.PUB_RESTRICTIONS.toString(), new EncodableArrayOfFixedIntegerRanges(6, 2, [], false));
+    if (legacy) {
+      fields.put(TcfCaV1Field.VENDOR_EXPRESS_CONSENT.toString(), new EncodableOptimizedFixedRange([]));
+      fields.put(TcfCaV1Field.VENDOR_IMPLIED_CONSENT.toString(), new EncodableOptimizedFixedRange([]));
+      fields.put(TcfCaV1Field.PUB_RESTRICTIONS.toString(), new EncodableArrayOfFixedIntegerRanges(6, 2, [], false));
+    } else {
+      fields.put(TcfCaV1Field.VENDOR_EXPRESS_CONSENT.toString(), new EncodableOptimizedFibonacciRange([]));
+      fields.put(TcfCaV1Field.VENDOR_IMPLIED_CONSENT.toString(), new EncodableOptimizedFibonacciRange([]));
+      fields.put(
+        TcfCaV1Field.PUB_RESTRICTIONS.toString(),
+        new EncodableArrayOfOptimizedFibonacciRanges(6, 2, [], false)
+      );
+    }
 
     return fields;
   }
@@ -129,9 +150,47 @@ export class TcfCaV1CoreSegment extends AbstractLazilyEncodableSegment<Encodable
     }
     try {
       let bitString: string = this.base64UrlEncoder.decode(encodedString);
+
+      // Prefer the spec-compliant (Fibonacci OptimizedRange) interpretation. If that doesn't
+      // round-trip back to the input, fall back to the legacy (fixed-range) interpretation used by
+      // the pre-fix encoder. This keeps older strings decodable; re-encoding always migrates them
+      // to the spec-compliant format because the fields decode into the Fibonacci datatypes.
+      if (this.tryDecode(bitString, fields, false)) {
+        return;
+      }
+      if (this.tryDecode(bitString, fields, true)) {
+        return;
+      }
+
+      // Neither interpretation round-trips cleanly; decode with the current interpretation as a
+      // best effort and surface any decoding error.
       this.bitStringEncoder.decode(bitString, this.getFieldNames(), fields);
     } catch (e) {
       throw new DecodingError("Unable to decode TcfCaV1CoreSegment '" + encodedString + "'");
     }
   }
+
+  /**
+   * Attempts to decode bitString using either the current or legacy field set and verifies the
+   * result by re-encoding it: if the re-encoded bits are a prefix of the decoded bits (the tail
+   * being base64 padding), the interpretation produced the string. On success the decoded values
+   * are copied into targetFields (which always use the current encoders) so that any subsequent
+   * re-encode emits the spec-compliant format.
+   */
+  private tryDecode(bitString: string, targetFields: EncodableBitStringFields, legacy: boolean): boolean {
+    try {
+      let candidate = this.buildFields(legacy);
+      this.bitStringEncoder.decode(bitString, this.getFieldNames(), candidate);
+      let reEncoded = this.bitStringEncoder.encode(candidate, this.getFieldNames());
+      if (bitString.startsWith(reEncoded)) {
+        for (let fieldName of this.getFieldNames()) {
+          targetFields.get(fieldName).setValue(candidate.get(fieldName).getValue());
+        }
+        return true;
+      }
+    } catch (e) {
+      // This interpretation does not apply; the caller will try the next one.
+    }
+    return false;
+  }
 }

modules/cmpapi/src/encoder/segment/TcfCaV1DisclosedVendorsSegment.ts

@@ -1,7 +1,8 @@
 import { AbstractBase64UrlEncoder } from "../base64/AbstractBase64UrlEncoder.js";
-import { TraditionalBase64UrlEncoder } from "../base64/TraditionalBase64UrlEncoder.js";
+import { CompressedBase64UrlEncoder } from "../base64/CompressedBase64UrlEncoder.js";
 import { BitStringEncoder } from "../bitstring/BitStringEncoder.js";
 import { EncodableFixedInteger } from "../datatype/EncodableFixedInteger.js";
+import { EncodableOptimizedFibonacciRange } from "../datatype/EncodableOptimizedFibonacciRange.js";
 import { EncodableOptimizedFixedRange } from "../datatype/EncodableOptimizedFixedRange.js";
 import { DecodingError } from "../error/DecodingError.js";
 import { EncodableBitStringFields } from "../field/EncodableBitStringFields.js";
@@ -10,7 +11,7 @@ import { TcfCaV1Field } from "../field/TcfCaV1Field.js";
 import { AbstractLazilyEncodableSegment } from "./AbstractLazilyEncodableSegment.js";
 
 export class TcfCaV1DisclosedVendorsSegment extends AbstractLazilyEncodableSegment<EncodableBitStringFields> {
-  private base64UrlEncoder: AbstractBase64UrlEncoder = TraditionalBase64UrlEncoder.getInstance();
+  private base64UrlEncoder: AbstractBase64UrlEncoder = CompressedBase64UrlEncoder.getInstance();
   private bitStringEncoder: BitStringEncoder = BitStringEncoder.getInstance();
 
   constructor(encodedString?: string) {
@@ -27,9 +28,23 @@ export class TcfCaV1DisclosedVendorsSegment extends AbstractLazilyEncodableSegme
 
   // overriden
   protected initializeFields(): EncodableBitStringFields {
+    return this.buildFields(false);
+  }
+
+  /**
+   * Builds the disclosed-vendors field set. When legacy is true the OptimizedRange field uses the
+   * pre-fix fixed-integer encoder; otherwise it uses the spec-compliant Fibonacci encoder. The
+   * legacy field set is only used to decode strings produced by the older encoder (see
+   * decodeSegment).
+   */
+  private buildFields(legacy: boolean): EncodableBitStringFields {
     let fields: EncodableBitStringFields = new EncodableBitStringFields();
     fields.put(TcfCaV1Field.DISCLOSED_VENDORS_SEGMENT_TYPE.toString(), new EncodableFixedInteger(3, 1));
-    fields.put(TcfCaV1Field.DISCLOSED_VENDORS.toString(), new EncodableOptimizedFixedRange([]));
+    if (legacy) {
+      fields.put(TcfCaV1Field.DISCLOSED_VENDORS.toString(), new EncodableOptimizedFixedRange([]));
+    } else {
+      fields.put(TcfCaV1Field.DISCLOSED_VENDORS.toString(), new EncodableOptimizedFibonacciRange([]));
+    }
     return fields;
   }
 
@@ -47,9 +62,44 @@ export class TcfCaV1DisclosedVendorsSegment extends AbstractLazilyEncodableSegme
     }
     try {
       let bitString: string = this.base64UrlEncoder.decode(encodedString);
+
+      // Prefer the spec-compliant (Fibonacci OptimizedRange) interpretation, falling back to the
+      // legacy (fixed-range) interpretation used by the pre-fix encoder. Re-encoding always
+      // migrates to the spec-compliant format because the values decode into the Fibonacci datatype.
+      if (this.tryDecode(bitString, fields, false)) {
+        return;
+      }
+      if (this.tryDecode(bitString, fields, true)) {
+        return;
+      }
+
       this.bitStringEncoder.decode(bitString, this.getFieldNames(), fields);
     } catch (e) {
-      throw new DecodingError("Unable to decode HeaderV1CoreSegment '" + encodedString + "'");
+      throw new DecodingError("Unable to decode TcfCaV1DisclosedVendorsSegment '" + encodedString + "'");
+    }
+  }
+
+  /**
+   * Attempts to decode bitString using either the current or legacy field set and verifies the
+   * result by re-encoding it: if the re-encoded bits are a prefix of the decoded bits (the tail
+   * being base64 padding), the interpretation produced the string. On success the decoded values
+   * are copied into targetFields (which always use the current encoders) so that any subsequent
+   * re-encode emits the spec-compliant format.
+   */
+  private tryDecode(bitString: string, targetFields: EncodableBitStringFields, legacy: boolean): boolean {
+    try {
+      let candidate = this.buildFields(legacy);
+      this.bitStringEncoder.decode(bitString, this.getFieldNames(), candidate);
+      let reEncoded = this.bitStringEncoder.encode(candidate, this.getFieldNames());
+      if (bitString.startsWith(reEncoded)) {
+        for (let fieldName of this.getFieldNames()) {
+          targetFields.get(fieldName).setValue(candidate.get(fieldName).getValue());
+        }
+        return true;
+      }
+    } catch (e) {
+      // This interpretation does not apply; the caller will try the next one.
     }
+    return false;
   }
 }

modules/cmpapi/test/GppModel.test.ts

@@ -394,7 +394,7 @@ describe("manifest.GppModel", (): void => {
 
     let gppString = gppModel.encode();
     expect(gppString).to.eql(
-      "DBACOeA~CPSG_8APSG_8ANwAAAENAwCgAAAAAAAAAAAAAAAAAAAA.IAAA.YAAAAAAAAAAA~BPSG_8APSG_8AAyACAENGdCgf_gfgAfgfgBgABABAAABAB4AACACAAA.fHHHA4444ao~1YNN"
+      "DBACOeA~CPSG_8APSG_8ANwAAAENAwCgAAAAAAAAAAAAAAAAAAAA.IAAA.YAAAAAAAAAAA~BPSG_8APSG_8AAyACAENGdCgf_gfgAfgfgBhADVqxGAD0AILVgAA.fHHHA4444ao~1YNN"
     );
 
     expect(gppString.split("~").length).to.eql(4);
@@ -905,12 +905,12 @@ describe("manifest.GppModel", (): void => {
 
   it("should fail to decode missing sections", (): void => {
     let gppModel = new GppModel(
-      "DBACOeA~CPSG_8APSG_8ANwAAAENAwCAAAAAAAAAAAAAAAAAAAAA.QAAA.IAAA~BPSG_8APSG_8AAyACAENGdCgf_gfgAfgfgBgABABAAABAB4AACACAAA.fHHHA4444ao"
+      "DBACOeA~CPSG_8APSG_8ANwAAAENAwCAAAAAAAAAAAAAAAAAAAAA.QAAA.IAAA~BPSG_8APSG_8AAyACAENGdCgf_gfgAfgfgBhADVqxGAD0AILVgAA.fHHHA4444ao"
     );
     expect(function () {
       gppModel.getHeader();
     }).to.throw(
-      "Unable to decode 'DBACOeA~CPSG_8APSG_8ANwAAAENAwCAAAAAAAAAAAAAAAAAAAAA.QAAA.IAAA~BPSG_8APSG_8AAyACAENGdCgf_gfgAfgfgBgABABAAABAB4AACACAAA.fHHHA4444ao'. The number of sections does not match the number of sections defined in the header."
+      "Unable to decode 'DBACOeA~CPSG_8APSG_8ANwAAAENAwCAAAAAAAAAAAAAAAAAAAAA.QAAA.IAAA~BPSG_8APSG_8AAyACAENGdCgf_gfgAfgfgBhADVqxGAD0AILVgAA.fHHHA4444ao'. The number of sections does not match the number of sections defined in the header."
     );
   });