From a776f3e80e0c23559deb9fd4c89106a973d731f2 Mon Sep 17 00:00:00 2001 From: treeform Date: Mon, 25 May 2026 10:53:21 -0700 Subject: [PATCH] Add flatty integer width modes --- .github/workflows/build.yml | 4 ++ README.md | 6 ++ src/flatty.nim | 128 ++++++++++++++++++++++++++---------- tests/test_modes.nim | 42 ++++++++++++ tests/tests.nim | 2 +- 5 files changed, 146 insertions(+), 36 deletions(-) create mode 100644 tests/test_modes.nim diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b5eaaeb..64da801 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,10 @@ jobs: nimby install -g "${{ github.event.repository.name }}/${{ github.event.repository.name }}.nimble" - name: Run native tests run: nim r tests/tests.nim + - name: Run flatty32 tests + run: nim r -d:flatty32 tests/tests.nim + - name: Run flatty64 tests + run: nim r -d:flatty64 tests/tests.nim - name: Run JavaScript tests with Node run: nim js -r tests/tests.nim - name: Run JavaScript tests in strict mode diff --git a/README.md b/README.md index 6b178b6..f7fee0c 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,12 @@ treeform/jsony .................... 11.199 ms 14.036 ms +/-1.773 x100 Flatty supports Nim's `js` mode. Some features like `uint64`/`int64` are supported badly because of Nim's limitations. Serializing of non-Nim JavaScript objects is not supported. +## Integer Width Modes + +By default, Flatty serializes Nim's default `int`, `uint`, and container lengths using the target's native `int` width. A 32-bit target uses 32-bit values, and a 64-bit target uses 64-bit values. + +You can force a specific width with `-d:flatty32` or `-d:flatty64`. Use the same mode when reading and writing a blob. Fixed-width types like `int32`, `uint32`, `int64`, and `uint64` are not affected by these modes. + ## Versioning Note, unlike `protobuf` or `thirft`, `flatty` has no versioning mechanism, if structure of your objects changes the resulting binary would be changed and could not be read back again. Because the schema is just plain Nim types, you need to make sure changing them does not impact your ability to read old flatty binary blobs. diff --git a/src/flatty.nim b/src/flatty.nim index 51c8f7c..4fbd302 100644 --- a/src/flatty.nim +++ b/src/flatty.nim @@ -12,6 +12,17 @@ else: type SomeTable*[K, V] = Table[K, V] | OrderedTable[K, V] type SomeSet[A] = set[A] | HashSet[A] | OrderedSet[A] +when defined(flatty32) and defined(flatty64): + {.error: "flatty32 and flatty64 cannot both be defined".} + +const FlattyIntSize = + when defined(flatty32): + 4 + elif defined(flatty64): + 8 + else: + sizeof(int) + when not defined(js): type UnsupportedHandle = AsyncFD | SocketHandle @@ -25,9 +36,36 @@ func skipsFlattyCopyMem[T](_: typedesc[T]): bool = else: T is UnsupportedHandle +func hasModeInt[T](_: typedesc[T]): bool = + when (T is int) or (T is uint): + true + elif T is distinct: + typeof(default(T).distinctBase).hasModeInt + elif T is object: + when default(T).isObjectVariant: + true + else: + block: + var found {.compileTime.} = false + for field in default(T).fields: + found = found or typeof(field).hasModeInt + found + elif T is tuple: + block: + var found {.compileTime.} = false + for field in default(T).fields: + found = found or typeof(field).hasModeInt + found + elif T is array: + typeof(default(T)[low(T)]).hasModeInt + else: + false + func copyable[T](_: typedesc[T]): bool = when not T.supportsCopyMem: false + elif FlattyIntSize != sizeof(int) and T.hasModeInt: + false elif skipsFlattyCopyMem(T): false elif (T is pointer) or (T is ptr) or (T is cstring) or (T is proc): @@ -54,6 +92,46 @@ func copyable[T](_: typedesc[T]): bool = else: true +template writeFlattyInt(s: var string, i: int, x: int) = + when FlattyIntSize == 4: + s.writeInt32(i, x.int32) + else: + s.writeInt64(i, x.int64) + +template addFlattyInt(s: var string, x: int) = + when FlattyIntSize == 4: + s.addInt32(x.int32) + else: + s.addInt64(x.int64) + +template addFlattyUInt(s: var string, x: uint) = + when FlattyIntSize == 4: + s.addUint32(x.uint32) + else: + s.addUint64(x.uint64) + +template readFlattyInt(s: string, i: var int): untyped = + block: + when FlattyIntSize == 4: + let value = s.readInt32(i).int + i += 4 + value + else: + let value = s.readInt64(i).int + i += 8 + value + +template readFlattyUInt(s: string, i: var int): untyped = + block: + when FlattyIntSize == 4: + let value = s.readUint32(i).uint + i += 4 + value + else: + let value = s.readUint64(i).uint + i += 8 + value + # Forward declarations. proc toFlatty*[T](s: var string, x: seq[T]) proc toFlatty*[T: object](s: var string, x: T) @@ -150,16 +228,10 @@ proc toFlatty*(s: var string, x: float32) = s.addFloat32(x) proc toFlatty*(s: var string, x: float64) = s.addFloat64(x) proc toFlatty*(s: var string, x: int) = - when sizeof(int) == 4: - s.addInt32(x) - else: - s.addInt64(x) + s.addFlattyInt(x) proc toFlatty*(s: var string, x: uint) = - when sizeof(int) == 4: - s.addUInt32(x) - else: - s.addUInt64(x) + s.addFlattyUInt(x) proc fromFlatty*(s: string, i: var int, x: var uint8) = x = s.readUint8(i) @@ -194,20 +266,10 @@ proc fromFlatty*(s: string, i: var int, x: var int64) = i += 8 proc fromFlatty*(s: string, i: var int, x: var int) = - when sizeof(int) == 4: - x = s.readInt32(i).int - i += 4 - else: - x = s.readInt64(i).int - i += 8 + x = s.readFlattyInt(i) proc fromFlatty*(s: string, i: var int, x: var uint) = - when sizeof(int) == 4: - x = s.readUInt32(i).uint - i += 4 - else: - x = s.readUInt64(i).uint - i += 8 + x = s.readFlattyUInt(i) proc fromFlatty*(s: string, i: var int, x: var float32) = x = s.readFloat32(i) @@ -227,12 +289,11 @@ proc fromFlatty*[T: enum and not range](s: string, i: var int, x: var T) = # Strings proc toFlatty*(s: var string, x: string) = - s.addInt64(x.len) + s.addFlattyInt(x.len) s.add(x) proc fromFlatty*(s: string, i: var int, x: var string) = - let len = s.readInt64(i).int - i += 8 + let len = s.readFlattyInt(i) when defined(js): x = s[i ..< i + len] else: @@ -247,18 +308,17 @@ proc toFlatty*[T](s: var string, x: seq[T]) = let oldLen = s.len byteLen = x.len * sizeof(T) - s.setLen(oldLen + 8 + byteLen) - s.writeInt64(oldLen, x.len.int64) + s.setLen(oldLen + FlattyIntSize + byteLen) + s.writeFlattyInt(oldLen, x.len) if byteLen > 0: - copyMem(s[oldLen + 8].addr, x[0].unsafeAddr, byteLen) + copyMem(s[oldLen + FlattyIntSize].addr, x[0].unsafeAddr, byteLen) else: - s.addInt64(x.len.int64) + s.addFlattyInt(x.len) for e in x: s.toFlatty(e) proc fromFlatty*[T](s: string, i: var int, x: var seq[T]) = - let len = s.readInt64(i).int - i += 8 + let len = s.readFlattyInt(i) when not defined(js) and T.copyable: when declared(setLenUninit): x.setLenUninit(len) @@ -309,7 +369,7 @@ proc fromFlatty*[T: distinct](s: string, i: var int, x: var T) = # Tables proc toTableLike[T](s: var string, K: type, V: type, x: T) {.inline.} = - s.addInt64(x.len.int64) + s.addFlattyInt(x.len) for k, v in x: s.toFlatty(k) s.toFlatty(v) @@ -317,8 +377,7 @@ proc toTableLike[T](s: var string, K: type, V: type, x: T) {.inline.} = proc fromTableLike[T]( s: string, i: var int, K: type, V: type, x: var T ) {.inline.} = - let len = s.readInt64(i).int - i += 8 + let len = s.readFlattyInt(i) when T is Table[K, V]: x = initTable[K, V](len) elif T is OrderedTable[K, V]: @@ -392,13 +451,12 @@ proc fromFlatty*[T](s: string, i: var int, x: var ref T) = # Sets proc toFlatty*[T](s: var string, x: SomeSet[T]) = - s.addInt64(x.card.int64) + s.addFlattyInt(x.card) for e in x: s.toFlatty(e) proc fromFlatty*[T](s: string, i: var int, x: var SomeSet[T]) = - let len = s.readInt64(i).int - i += 8 + let len = s.readFlattyInt(i) when x is HashSet[T]: x = initHashSet[T](len) elif x is OrderedSet[T]: diff --git a/tests/test_modes.nim b/tests/test_modes.nim new file mode 100644 index 0000000..73a8aa1 --- /dev/null +++ b/tests/test_modes.nim @@ -0,0 +1,42 @@ +import flatty + +type + ModeObj = object + a: int + b: uint + c: int16 + + ModeTuple = tuple + a: int + b: uint8 + c: uint + +const FlattyModeIntSize = + when defined(flatty32): + 4 + elif defined(flatty64): + 8 + else: + sizeof(int) + +doAssert 1.toFlatty.len == FlattyModeIntSize +doAssert 1.uint.toFlatty.len == FlattyModeIntSize + +doAssert "abc".toFlatty.len == FlattyModeIntSize + 3 +doAssert @[1.uint8, 2, 3].toFlatty.len == FlattyModeIntSize + 3 + +let modeObj = ModeObj(a: 1, b: 2.uint, c: 3.int16) +let modeTuple: ModeTuple = (a: 1, b: 2.uint8, c: 3.uint) + +doAssert @[1, 2, 3].toFlatty.len == FlattyModeIntSize + 3 * FlattyModeIntSize +doAssert [1, 2, 3].toFlatty.len == 3 * FlattyModeIntSize +doAssert modeObj.toFlatty.len == 2 * FlattyModeIntSize + 2 +doAssert modeTuple.toFlatty.len == 2 * FlattyModeIntSize + 1 + +doAssert 1.toFlatty.fromFlatty(int) == 1 +doAssert 1.uint.toFlatty.fromFlatty(uint) == 1.uint +doAssert "abc".toFlatty.fromFlatty(string) == "abc" +doAssert @[1, 2, 3].toFlatty.fromFlatty(seq[int]) == @[1, 2, 3] +doAssert [1, 2, 3].toFlatty.fromFlatty(array[3, int]) == [1, 2, 3] +doAssert modeObj.toFlatty.fromFlatty(ModeObj) == modeObj +doAssert modeTuple.toFlatty.fromFlatty(ModeTuple) == modeTuple diff --git a/tests/tests.nim b/tests/tests.nim index f442be1..8310be1 100644 --- a/tests/tests.nim +++ b/tests/tests.nim @@ -1,4 +1,4 @@ import test_binny, test_datetime, test_flatty, test_hashy, test_hexprint, - test_memoryused, test_objvar + test_memoryused, test_modes, test_objvar echo "all tests pass"