diff --git a/crates/wseg-build/src/aa_binfmt.rs b/crates/wseg-build/src/aa_binfmt.rs index 36bf331..14b819f 100644 --- a/crates/wseg-build/src/aa_binfmt.rs +++ b/crates/wseg-build/src/aa_binfmt.rs @@ -271,7 +271,10 @@ impl<'a> Posting<'a> { /// A decoded data attribute: schema field index + its canonical string value. pub type Attr = (u8, String); -const ASSET_VERSION: u8 = 1; +// v2 (Cycle B): asset FWD unchanged structurally; template FWD gains transferable/burnable/ +// max_supply/issued_supply; the new collection FWD (TABLE_AA_COLL_FWD) is introduced. The version byte +// gates every data blob, so a v2 reader fails closed on a v1 segment (rebuild required) and vice-versa. +const ASSET_VERSION: u8 = 2; /// Encode an asset forward record. `immutable` is non-empty only for assets without a template /// (templated assets get their immutable data from the template blob, joined at read time). @@ -299,12 +302,28 @@ pub fn encode_asset( o } -/// Encode a template forward record (its immutable attributes, stored once). -pub fn encode_template(template_id: i32, schema: u64, immutable: &[Attr]) -> Vec { - let mut o = Vec::with_capacity(16 + immutable.len() * 12); +/// Encode a template forward record (its immutable attributes, stored once) plus the v2 fields the full +/// API shape needs: `transferable`/`burnable` (0/1) and `max_supply`/`issued_supply` (uint32; 0 supply = +/// unlimited). Layout: `version | template_id(i32) | schema(u64) | transferable(u8) | burnable(u8) | +/// max_supply(u32) | issued_supply(u32) | immutable attrs` — attrs start at offset 23. +#[allow(clippy::too_many_arguments)] +pub fn encode_template( + template_id: i32, + schema: u64, + transferable: u8, + burnable: u8, + max_supply: u32, + issued_supply: u32, + immutable: &[Attr], +) -> Vec { + let mut o = Vec::with_capacity(25 + immutable.len() * 12); // 23 header + 2 attr-count prefix o.push(ASSET_VERSION); pi32(&mut o, template_id); pu64(&mut o, schema); + o.push(transferable); + o.push(burnable); + pu32(&mut o, max_supply); + pu32(&mut o, issued_supply); put_attrs(&mut o, immutable); o } @@ -368,13 +387,121 @@ pub fn decode_asset(b: &[u8]) -> AssetRec { } } -/// Decoded template forward record: (template_id, schema, immutable attrs). -pub fn decode_template(b: &[u8]) -> (i32, u64, Vec) { - let mut p = 1usize; +/// Decoded template forward record (v2). +#[derive(Debug, PartialEq)] +pub struct TemplateRec { + pub template_id: i32, + pub schema: u64, + pub transferable: u8, + pub burnable: u8, + pub max_supply: u32, + pub issued_supply: u32, + pub immutable: Vec, +} + +pub fn decode_template(b: &[u8]) -> TemplateRec { + assert_eq!( + b[0], ASSET_VERSION, + "decode_template: version byte mismatch (rebuild the segment)" + ); + let mut p = 1usize; // skip version let template_id = gi32(b, &mut p); let schema = gu64(b, &mut p); + let transferable = b[p]; + p += 1; + let burnable = b[p]; + p += 1; + let max_supply = gu32(b, &mut p); + let issued_supply = gu32(b, &mut p); let immutable = get_attrs(b, &mut p); - (template_id, schema, immutable) + TemplateRec { + template_id, + schema, + transferable, + burnable, + max_supply, + issued_supply, + immutable, + } +} + +// ── collection forward record (TABLE_AA_COLL_FWD) ─────────────────────────────────────────────────── +/// Encode a collection forward record. Layout: `version | collection(u64) | author(u64) | +/// allow_notify(u8) | auth_count(u8) | authorized(u64×N) | notify_count(u8) | notify(u64×M) | +/// market_fee(f64) | data attrs`. `market_fee` is the contract `double` (stored as f64 to avoid the +/// precision loss an f32 would introduce in the parity diff). Account lists are capped at 255 each (u8 +/// count) — real collections carry a handful, far below the cap; the builder (`push_collection`) warns if +/// the cap ever bites, so the truncation is never silent. +#[allow(clippy::too_many_arguments)] +pub fn encode_collection( + collection: u64, + author: u64, + allow_notify: u8, + authorized: &[u64], + notify: &[u64], + market_fee: f64, + data: &[Attr], +) -> Vec { + let nauth = authorized.len().min(255); + let nnotify = notify.len().min(255); + let mut o = Vec::with_capacity(30 + (nauth + nnotify) * 8 + data.len() * 12); // 28 fixed + 2 attr-count + o.push(ASSET_VERSION); + pu64(&mut o, collection); + pu64(&mut o, author); + o.push(allow_notify); + o.push(nauth as u8); + for &acc in &authorized[..nauth] { + pu64(&mut o, acc); + } + o.push(nnotify as u8); + for &acc in ¬ify[..nnotify] { + pu64(&mut o, acc); + } + o.extend_from_slice(&market_fee.to_le_bytes()); + put_attrs(&mut o, data); + o +} + +/// Decoded collection forward record. +#[derive(Debug, PartialEq)] +pub struct CollectionRec { + pub collection: u64, + pub author: u64, + pub allow_notify: u8, + pub authorized: Vec, + pub notify: Vec, + pub market_fee: f64, + pub data: Vec, +} + +pub fn decode_collection(b: &[u8]) -> CollectionRec { + assert_eq!( + b[0], ASSET_VERSION, + "decode_collection: version byte mismatch (rebuild the segment)" + ); + let mut p = 1usize; // skip version + let collection = gu64(b, &mut p); + let author = gu64(b, &mut p); + let allow_notify = b[p]; + p += 1; + let nauth = b[p] as usize; + p += 1; + let authorized: Vec = (0..nauth).map(|_| gu64(b, &mut p)).collect(); + let nnotify = b[p] as usize; + p += 1; + let notify: Vec = (0..nnotify).map(|_| gu64(b, &mut p)).collect(); + let market_fee = f64::from_le_bytes(b[p..p + 8].try_into().unwrap()); + p += 8; + let data = get_attrs(b, &mut p); + CollectionRec { + collection, + author, + allow_notify, + authorized, + notify, + market_fee, + data, + } } // ── config singleton (TABLE_AA_CONFIG) ───────────────────────────────────────────────────────────── @@ -474,7 +601,7 @@ mod tests { #[test] fn golden_asset_record() { let golden: &[u8] = &[ - 1, // version + 2, // version (Cycle B = 2) 1, 0, 0, 0, 0, 0, 0, 0, // owner = 1 2, 0, 0, 0, 0, 0, 0, 0, // collection = 2 3, 0, 0, 0, 0, 0, 0, 0, // schema = 3 @@ -487,6 +614,59 @@ mod tests { assert_eq!(encode_asset(1, 2, 3, 7, 100, 42, &[], &[]), golden); } + /// Cross-repo byte golden for the v2 template FWD record (must match GOLDEN_TEMPLATE_V2 in + /// wormdb-domain-atomicassets/src/binfmt.zig). Raw field values (not name-encoded) so the byte array + /// is mirrored verbatim on both sides — the same lock as `golden_asset_record`. + #[test] + fn golden_template_v2_record() { + let golden: &[u8] = &[ + 2, // version + 3, 0, 0, 0, // template_id = 3 + 99, 0, 0, 0, 0, 0, 0, 0, // schema = 99 + 1, // transferable = 1 + 0, // burnable = 0 + 0, 0, 0, 0, // max_supply = 0 (unlimited) + 5, 0, 0, 0, // issued_supply = 5 + 1, 0, // immutable attr count = 1 + 0, // field_idx = 0 + 9, 0, // val_len = 9 + 67, 104, 97, 114, 105, 122, 97, 114, 100, // "Charizard" + ]; + let attrs = vec![(0u8, "Charizard".to_string())]; + assert_eq!( + encode_template(3, 99, 1, 0, 0, 5, &attrs), + golden, + "template v2 byte layout drift — update GOLDEN_TEMPLATE_V2 in the Zig reader too" + ); + } + + /// Cross-repo byte golden for the collection FWD record (must match GOLDEN_COLLECTION_V2 in the Zig + /// reader). Raw field values; market_fee=0.0 (8 zero bytes) keeps the array unambiguous — the f64 + /// value path is covered by `collection_record_round_trips`. One authorized account, no notify. + #[test] + fn golden_collection_v2_record() { + let golden: &[u8] = &[ + 2, // version + 77, 0, 0, 0, 0, 0, 0, 0, // collection = 77 + 88, 0, 0, 0, 0, 0, 0, 0, // author = 88 + 1, // allow_notify = 1 + 1, // auth_count = 1 + 66, 0, 0, 0, 0, 0, 0, 0, // authorized[0] = 66 + 0, // notify_count = 0 + 0, 0, 0, 0, 0, 0, 0, 0, // market_fee = 0.0 (f64 LE) + 1, 0, // data attr count = 1 + 0, // field_idx = 0 (name) + 5, 0, // val_len = 5 + 77, 121, 67, 111, 108, // "MyCol" + ]; + let data = vec![(0u8, "MyCol".to_string())]; + assert_eq!( + encode_collection(77, 88, 1, &[66], &[], 0.0, &data), + golden, + "collection byte layout drift — update GOLDEN_COLLECTION_V2 in the Zig reader too" + ); + } + #[test] fn type_tag_round_trips_incl_arrays() { for t in ["uint64", "string", "double", "ipfs", "image", "bool"] { @@ -586,11 +766,38 @@ mod tests { #[test] fn template_record_round_trips() { let immutable = vec![(0u8, "Charizard".to_string()), (2u8, "150".to_string())]; - let blob = encode_template(3, crate::name::encode("pokemon"), &immutable); - let (tid, schema, attrs) = decode_template(&blob); - assert_eq!(tid, 3); - assert_eq!(schema, crate::name::encode("pokemon")); - assert_eq!(attrs, immutable); + let blob = encode_template(3, crate::name::encode("pokemon"), 1, 0, 100, 42, &immutable); + let t = decode_template(&blob); + assert_eq!(t.template_id, 3); + assert_eq!(t.schema, crate::name::encode("pokemon")); + assert_eq!(t.transferable, 1); + assert_eq!(t.burnable, 0); + assert_eq!(t.max_supply, 100); + assert_eq!(t.issued_supply, 42); + assert_eq!(t.immutable, immutable); + } + + #[test] + fn collection_record_round_trips() { + let data = vec![(0u8, "MyCol".to_string()), (1u8, "Qmimg".to_string())]; + let auth = vec![crate::name::encode("alice"), crate::name::encode("bob")]; + let blob = encode_collection( + crate::name::encode("mycol"), + crate::name::encode("alice"), + 1, + &auth, + &[], + 0.06, + &data, + ); + let c = decode_collection(&blob); + assert_eq!(c.collection, crate::name::encode("mycol")); + assert_eq!(c.author, crate::name::encode("alice")); + assert_eq!(c.allow_notify, 1); + assert_eq!(c.authorized, auth); + assert!(c.notify.is_empty()); + assert_eq!(c.market_fee, 0.06); + assert_eq!(c.data, data); } #[test] diff --git a/crates/wseg-build/src/aa_builder.rs b/crates/wseg-build/src/aa_builder.rs index b2967e8..b531dc0 100644 --- a/crates/wseg-build/src/aa_builder.rs +++ b/crates/wseg-build/src/aa_builder.rs @@ -9,7 +9,8 @@ use std::collections::HashMap; use mongodb::bson::{Bson, Document}; use crate::aa_binfmt::{ - encode_asset, encode_config, encode_posting_hybrid, encode_schema_format, encode_template, Attr, + encode_asset, encode_collection, encode_config, encode_posting_hybrid, encode_schema_format, + encode_template, Attr, }; use crate::aa_tables::*; use crate::name; @@ -20,6 +21,7 @@ use crate::wseg::{write_segment, IndexEntry, Table}; pub struct AaStats { pub schemas: u64, pub templates: u64, + pub collections: u64, pub assets: u64, pub bytes: u64, } @@ -59,6 +61,10 @@ pub struct AtomicBuilder { asset_fwd: Fwd, tmpl_fwd: Fwd, schema_fwd: Fwd, + coll_fwd: Fwd, + /// Global collection-data field → index (from config `collection_format`), to encode collection + /// `data` attrs compactly (same idx-keyed scheme as asset/template attrs). Set by `push_config`. + coll_format_idx: HashMap, by_owner: HashMap>, by_coll: HashMap>, by_schema: HashMap>, @@ -85,6 +91,8 @@ impl AtomicBuilder { asset_fwd: Fwd::default(), tmpl_fwd: Fwd::default(), schema_fwd: Fwd::default(), + coll_fwd: Fwd::default(), + coll_format_idx: HashMap::new(), by_owner: HashMap::new(), by_coll: HashMap::new(), by_schema: HashMap::new(), @@ -101,6 +109,7 @@ impl AtomicBuilder { pub fn push(&mut self, coll: &str, d: &Document) { match coll { "atomicassets-config" => self.push_config(d), + "atomicassets-collections" => self.push_collection(d), "atomicassets-schemas" => self.push_schema(d), "atomicassets-templates" => self.push_template(d), "atomicassets-assets" => self.push_asset(d), @@ -122,6 +131,14 @@ impl AtomicBuilder { } } } + // Global collection-data field→index, for compact collection `data` attr encoding (push_collection). + // The attr field_idx is a u8, so fields past 255 are unrepresentable — skip them (never happens for + // a real collection_format, but avoids a silent `as u8` wrap). + self.coll_format_idx = fmt + .iter() + .enumerate() + .filter_map(|(i, (n, _))| u8::try_from(i).ok().map(|ix| (n.clone(), ix))) + .collect(); let mut tokens: Vec<(u64, String, i64)> = Vec::new(); if let Ok(arr) = d.get_array("supported_tokens") { for t in arr { @@ -172,11 +189,56 @@ impl AtomicBuilder { }; let skey = coll_schema_key(coll, sch); let immutable = self.attrs(skey, d.get_document("immutable_data").ok()); - let blob = encode_template(tid, name::encode(sch), &immutable); + let blob = encode_template( + tid, + name::encode(sch), + doc_bool(d, "transferable") as u8, + doc_bool(d, "burnable") as u8, + doc_u32(d, "max_supply"), + doc_u32(d, "issued_supply"), + &immutable, + ); self.tmpl_fwd.push(template_key(tid as i64), &blob); self.stats.templates += 1; } + /// `atomicassets-collections` → the collection forward record (TABLE_AA_COLL_FWD), keyed by the + /// name-encoded collection. `data` attrs are encoded against the global config `collection_format`. + fn push_collection(&mut self, d: &Document) { + let Some(coll) = doc_str(d, "collection_name") else { + return; + }; + let author = doc_str(d, "author").map(name::encode).unwrap_or(0); + let authorized = doc_name_array(d, "authorized_accounts"); + let notify = doc_name_array(d, "notify_accounts"); + if authorized.len() > 255 || notify.len() > 255 { + eprintln!( + "[aa-build] WARN collection {coll}: account list exceeds the 255 cap (authorized={}, notify={}) — truncating", + authorized.len(), + notify.len() + ); + } + // collection `data` attrs keyed by the global collection_format index (mirrors asset/template). + let data: Vec = match d.get_document("data").ok() { + Some(doc) => doc + .iter() + .filter_map(|(k, v)| self.coll_format_idx.get(k).map(|&i| (i, bson_canon(v)))) + .collect(), + None => Vec::new(), + }; + let blob = encode_collection( + collection_key(coll), + author, + doc_bool(d, "allow_notify") as u8, + &authorized, + ¬ify, + doc_f64(d, "market_fee"), + &data, + ); + self.coll_fwd.push(collection_key(coll), &blob); + self.stats.collections += 1; + } + fn push_asset(&mut self, d: &Document) { let (Some(coll), Some(sch), Some(owner)) = ( doc_str(d, "collection_name"), @@ -248,15 +310,61 @@ impl AtomicBuilder { self.stats.schemas += 1; } - /// Register a template forward record (its immutable attrs, stored once). - pub fn push_template_raw(&mut self, template_id: i32, schema_u: u64, immutable: &[Attr]) { + /// Register a template forward record (its immutable attrs + v2 fields, stored once). + #[allow(clippy::too_many_arguments)] + pub fn push_template_raw( + &mut self, + template_id: i32, + schema_u: u64, + transferable: u8, + burnable: u8, + max_supply: u32, + issued_supply: u32, + immutable: &[Attr], + ) { self.tmpl_fwd.push( template_key(template_id as i64), - &encode_template(template_id, schema_u, immutable), + &encode_template( + template_id, + schema_u, + transferable, + burnable, + max_supply, + issued_supply, + immutable, + ), ); self.stats.templates += 1; } + /// Register a collection forward record (already-decoded fields), keyed by the name-encoded + /// collection. Used by compaction to carry COLL_FWD over into a fresh segment. + #[allow(clippy::too_many_arguments)] + pub fn push_collection_raw( + &mut self, + collection: u64, + author: u64, + allow_notify: u8, + authorized: &[u64], + notify: &[u64], + market_fee: f64, + data: &[Attr], + ) { + self.coll_fwd.push( + collection, + &encode_collection( + collection, + author, + allow_notify, + authorized, + notify, + market_fee, + data, + ), + ); + self.stats.collections += 1; + } + /// Add one already-decoded current asset to the forward store + every inverted index. `facet_keys` /// are the precomputed `data_attr_key`s the asset participates in (0 or 1 per configured facet). #[allow(clippy::too_many_arguments)] @@ -359,6 +467,7 @@ impl AtomicBuilder { std::mem::take(&mut self.asset_fwd).into_table(TABLE_AA_FWD), std::mem::take(&mut self.tmpl_fwd).into_table(TABLE_AA_TMPL_FWD), std::mem::take(&mut self.schema_fwd).into_table(TABLE_AA_SCHEMAS), + std::mem::take(&mut self.coll_fwd).into_table(TABLE_AA_COLL_FWD), posting_table(TABLE_AA_BY_OWNER, std::mem::take(&mut self.by_owner)), posting_table(TABLE_AA_BY_COLL, std::mem::take(&mut self.by_coll)), posting_table(TABLE_AA_BY_SCHEMA, std::mem::take(&mut self.by_schema)), @@ -455,6 +564,32 @@ fn doc_i64(d: &Document, k: &str) -> Option { _ => None, } } +fn doc_bool(d: &Document, k: &str) -> bool { + match d.get(k) { + Some(Bson::Boolean(v)) => *v, + Some(Bson::Int32(i)) => *i != 0, + Some(Bson::Int64(i)) => *i != 0, + _ => false, + } +} +fn doc_f64(d: &Document, k: &str) -> f64 { + match d.get(k) { + Some(Bson::Double(f)) => *f, + Some(Bson::Int32(i)) => *i as f64, + Some(Bson::Int64(i)) => *i as f64, + _ => 0.0, + } +} +/// Collect a BSON string array (e.g. authorized/notify accounts) as name-encoded u64s. +fn doc_name_array(d: &Document, k: &str) -> Vec { + match d.get_array(k) { + Ok(arr) => arr + .iter() + .filter_map(|b| b.as_str().map(name::encode)) + .collect(), + Err(_) => Vec::new(), + } +} /// Canonical string form of a bson value, matching what the API filters on. fn bson_canon(b: &Bson) -> String { diff --git a/crates/wseg-build/src/aa_live.rs b/crates/wseg-build/src/aa_live.rs index acc4e8f..c82af53 100644 --- a/crates/wseg-build/src/aa_live.rs +++ b/crates/wseg-build/src/aa_live.rs @@ -19,7 +19,9 @@ use std::sync::Arc; use memmap2::Mmap; use roaring::RoaringTreemap; -use crate::aa_binfmt::{decode_asset, decode_schema_format, decode_template, Attr, Posting}; +use crate::aa_binfmt::{ + decode_asset, decode_collection, decode_schema_format, decode_template, Attr, Posting, +}; use crate::aa_builder::{AaStats, AtomicBuilder}; use crate::aa_tables::*; use crate::name; @@ -1107,8 +1109,29 @@ impl LiveSeg { b.push_schema_raw(key, &decode_schema_format(blob)); }); self.base.for_each_entry(TABLE_AA_TMPL_FWD, |_k, blob| { - let (tid, schema_u, immut) = decode_template(blob); - b.push_template_raw(tid, schema_u, &immut); + let t = decode_template(blob); + b.push_template_raw( + t.template_id, + t.schema, + t.transferable, + t.burnable, + t.max_supply, + t.issued_supply, + &t.immutable, + ); + }); + // collection forward records carry over too (v2 TABLE_AA_COLL_FWD) — else compaction drops them. + self.base.for_each_entry(TABLE_AA_COLL_FWD, |_k, blob| { + let c = decode_collection(blob); + b.push_collection_raw( + c.collection, + c.author, + c.allow_notify, + &c.authorized, + &c.notify, + c.market_fee, + &c.data, + ); }); // every CURRENT asset, exactly once: base assets (overlay-current, tombstones dropped) … diff --git a/crates/wseg-build/src/bin/aa_build.rs b/crates/wseg-build/src/bin/aa_build.rs index 26c0f71..5d87173 100644 --- a/crates/wseg-build/src/bin/aa_build.rs +++ b/crates/wseg-build/src/bin/aa_build.rs @@ -90,14 +90,26 @@ async fn main() -> Result<()> { let t0 = Instant::now(); let mut b = AtomicBuilder::new(fields); - // config first (singleton — contract/collection_format/supported_tokens), then schemas (need + // config first (singleton — contract/collection_format/supported_tokens; sets the collection_format + // index), then collections (depend on that index for their `data` attrs), then schemas (need // `format`), then templates, then assets (projected). stream(&db, "atomicassets-config", doc! {}, 0, &mut b, t0).await?; + stream( + &db, + "atomicassets-collections", + doc! { "collection_name": 1, "author": 1, "allow_notify": 1, "authorized_accounts": 1, + "notify_accounts": 1, "market_fee": 1, "data": 1 }, + 0, + &mut b, + t0, + ) + .await?; stream(&db, "atomicassets-schemas", doc! {}, 0, &mut b, t0).await?; stream( &db, "atomicassets-templates", - doc! { "collection_name": 1, "schema_name": 1, "template_id": 1, "immutable_data": 1 }, + doc! { "collection_name": 1, "schema_name": 1, "template_id": 1, "immutable_data": 1, + "transferable": 1, "burnable": 1, "max_supply": 1, "issued_supply": 1 }, 0, &mut b, t0, @@ -118,11 +130,12 @@ async fn main() -> Result<()> { let stats = b.finish(&args.out).context("write segment")?; let sz = std::fs::metadata(&args.out)?.len(); eprintln!( - "[aa-build] wrote {} | {} assets, {} templates, {} schemas | {} MiB ({} bytes) in {:.0}s", + "[aa-build] wrote {} | {} assets, {} templates, {} schemas, {} collections | {} MiB ({} bytes) in {:.0}s", args.out, stats.assets, stats.templates, stats.schemas, + stats.collections, sz >> 20, sz, t0.elapsed().as_secs_f64()