From 39d4f0a1f388bb4105f4c4c4a560975ce1247420 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Wed, 8 Feb 2023 13:00:18 +0530 Subject: [PATCH 01/96] Add BlobSidecar type --- consensus/types/src/blob_sidecar.rs | 51 +++++++++++++++++++++++++++++ consensus/types/src/lib.rs | 4 ++- crypto/kzg/src/kzg_commitment.rs | 6 ++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 consensus/types/src/blob_sidecar.rs diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs new file mode 100644 index 00000000000..3a2012839e6 --- /dev/null +++ b/consensus/types/src/blob_sidecar.rs @@ -0,0 +1,51 @@ +use crate::test_utils::TestRandom; +use crate::{Blob, EthSpec, Hash256, SignedRoot, Slot}; +use derivative::Derivative; +use kzg::{KzgCommitment, KzgProof}; +use serde_derive::{Deserialize, Serialize}; +use ssz::Encode; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + Default, + TestRandom, + Derivative, + arbitrary::Arbitrary, +)] +#[serde(bound = "T: EthSpec")] +#[arbitrary(bound = "T: EthSpec")] +#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] +pub struct BlobSidecar { + pub block_root: Hash256, + // TODO: fix the type, should fit in u8 as well + pub index: u64, + pub slot: Slot, + pub block_parent_root: Hash256, + pub proposer_index: u64, + pub blob: Blob, + pub kzg_commitment: KzgCommitment, + pub kzg_proof: KzgProof, +} + +impl SignedRoot for BlobSidecar {} + +impl BlobSidecar { + pub fn empty() -> Self { + Self::default() + } + + #[allow(clippy::integer_arithmetic)] + pub fn max_size() -> usize { + // Fixed part + Self::empty().as_ssz_bytes().len() + } +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 380f5e527b8..4ec740eff4f 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -99,6 +99,7 @@ pub mod slot_data; #[cfg(feature = "sqlite")] pub mod sqlite; +pub mod blob_sidecar; pub mod blobs_sidecar; pub mod signed_block_and_blobs; pub mod transaction; @@ -121,7 +122,8 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; -pub use crate::blobs_sidecar::{Blobs, BlobsSidecar, KzgCommitments}; +pub use crate::blob_sidecar::BlobSidecar; +pub use crate::blobs_sidecar::BlobsSidecar; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; diff --git a/crypto/kzg/src/kzg_commitment.rs b/crypto/kzg/src/kzg_commitment.rs index 16fe1d8305a..7a43d7009d0 100644 --- a/crypto/kzg/src/kzg_commitment.rs +++ b/crypto/kzg/src/kzg_commitment.rs @@ -20,6 +20,12 @@ impl Display for KzgCommitment { } } +impl Default for KzgCommitment { + fn default() -> Self { + KzgCommitment([0; KZG_COMMITMENT_BYTES_LEN]) + } +} + impl TreeHash for KzgCommitment { fn tree_hash_type() -> tree_hash::TreeHashType { <[u8; KZG_COMMITMENT_BYTES_LEN] as TreeHash>::tree_hash_type() From bf40acd9dfb2aedc140777260db5d1bda3e13f7b Mon Sep 17 00:00:00 2001 From: Diva M Date: Mon, 6 Mar 2023 17:32:40 -0500 Subject: [PATCH 02/96] adjust constant to spec values and names --- beacon_node/lighthouse_network/src/rpc/config.rs | 2 +- beacon_node/lighthouse_network/src/rpc/methods.rs | 10 +++++++--- beacon_node/lighthouse_network/src/rpc/mod.rs | 4 ++-- .../network/src/beacon_processor/worker/rpc_methods.rs | 4 ++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/beacon_node/lighthouse_network/src/rpc/config.rs b/beacon_node/lighthouse_network/src/rpc/config.rs index 7d88dbaff77..cdee1d495a1 100644 --- a/beacon_node/lighthouse_network/src/rpc/config.rs +++ b/beacon_node/lighthouse_network/src/rpc/config.rs @@ -80,7 +80,7 @@ impl OutboundRateLimiterConfig { Quota::n_every(methods::MAX_REQUEST_BLOCKS, 10); pub const DEFAULT_BLOCKS_BY_ROOT_QUOTA: Quota = Quota::n_every(128, 10); pub const DEFAULT_BLOBS_BY_RANGE_QUOTA: Quota = - Quota::n_every(methods::MAX_REQUEST_BLOBS_SIDECARS, 10); + Quota::n_every(methods::MAX_REQUEST_BLOB_SIDECARS, 10); pub const DEFAULT_BLOBS_BY_ROOT_QUOTA: Quota = Quota::n_every(128, 10); } diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index 088cf90fa98..2ee4cc97987 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -5,7 +5,7 @@ use regex::bytes::Regex; use serde::Serialize; use ssz_derive::{Decode, Encode}; use ssz_types::{ - typenum::{U1024, U256}, + typenum::{U1024, U256, U512}, VariableList, }; use std::ops::Deref; @@ -26,8 +26,12 @@ pub const MAX_REQUEST_BLOCKS: u64 = 1024; pub type MaxErrorLen = U256; pub const MAX_ERROR_LEN: u64 = 256; -pub type MaxRequestBlobsSidecars = U1024; -pub const MAX_REQUEST_BLOBS_SIDECARS: u64 = 1024; +// TODO: this is calculated as MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK and +// MAX_BLOBS_PER_BLOCK comes from the spec. +// MAX_REQUEST_BLOCKS_DENEB = 128 +// MAX_BLOBS_PER_BLOCK = 4 +pub type MaxRequestBlobSidecars = U512; +pub const MAX_REQUEST_BLOB_SIDECARS: u64 = 512; /// Wrapper over SSZ List to represent error message in rpc responses. #[derive(Debug, Clone)] diff --git a/beacon_node/lighthouse_network/src/rpc/mod.rs b/beacon_node/lighthouse_network/src/rpc/mod.rs index 8727f7df764..f2abb8821a9 100644 --- a/beacon_node/lighthouse_network/src/rpc/mod.rs +++ b/beacon_node/lighthouse_network/src/rpc/mod.rs @@ -24,7 +24,7 @@ pub(crate) use handler::HandlerErr; pub(crate) use methods::{MetaData, MetaDataV1, MetaDataV2, Ping, RPCCodedResponse, RPCResponse}; pub(crate) use protocol::{InboundRequest, RPCProtocol}; -use crate::rpc::methods::MAX_REQUEST_BLOBS_SIDECARS; +use crate::rpc::methods::MAX_REQUEST_BLOB_SIDECARS; pub use handler::SubstreamId; pub use methods::{ BlocksByRangeRequest, BlocksByRootRequest, GoodbyeReason, LightClientBootstrapRequest, @@ -148,7 +148,7 @@ impl RPC { .n_every(Protocol::BlobsByRoot, 128, Duration::from_secs(10)) .n_every( Protocol::BlobsByRange, - MAX_REQUEST_BLOBS_SIDECARS, + MAX_REQUEST_BLOB_SIDECARS, Duration::from_secs(10), ) .build() diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index cefb1c77e16..ad0595ea05b 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -5,7 +5,7 @@ use crate::sync::SyncMessage; use beacon_chain::{BeaconChainError, BeaconChainTypes, HistoricalBlockError, WhenSlotSkipped}; use itertools::process_results; use lighthouse_network::rpc::methods::{ - BlobsByRangeRequest, BlobsByRootRequest, MAX_REQUEST_BLOBS_SIDECARS, + BlobsByRangeRequest, BlobsByRootRequest, MAX_REQUEST_BLOB_SIDECARS, }; use lighthouse_network::rpc::StatusMessage; use lighthouse_network::rpc::*; @@ -669,7 +669,7 @@ impl Worker { ); // Should not send more than max request blocks - if req.count > MAX_REQUEST_BLOBS_SIDECARS { + if req.count > MAX_REQUEST_BLOB_SIDECARS { return self.send_error_response( peer_id, RPCResponseErrorCode::InvalidRequest, From 75a0d52ee0804d51c0401155a8515501744b02fa Mon Sep 17 00:00:00 2001 From: Diva M Date: Mon, 6 Mar 2023 17:58:00 -0500 Subject: [PATCH 03/96] get back the Blobs pub use to fix compilation issues --- consensus/types/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 4ec740eff4f..4d69888b770 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -123,7 +123,7 @@ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; pub use crate::blob_sidecar::BlobSidecar; -pub use crate::blobs_sidecar::BlobsSidecar; +pub use crate::blobs_sidecar::{BlobsSidecar, Blobs}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; From 63011c5bcadeecc5a41494d1b9cade3b3bb559ad Mon Sep 17 00:00:00 2001 From: Diva M Date: Mon, 6 Mar 2023 18:09:31 -0500 Subject: [PATCH 04/96] fmt --- consensus/types/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 4d69888b770..a2dfb853ead 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -123,7 +123,7 @@ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; pub use crate::blob_sidecar::BlobSidecar; -pub use crate::blobs_sidecar::{BlobsSidecar, Blobs}; +pub use crate::blobs_sidecar::{Blobs, BlobsSidecar}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; From 545532a883d6ae2a5850a220945b13111ae79c26 Mon Sep 17 00:00:00 2001 From: Divma <26765164+divagant-martian@users.noreply.github.com> Date: Tue, 7 Mar 2023 16:28:45 -0500 Subject: [PATCH 05/96] fix rpc types to free the blobs (#4059) * rename to follow name in spec * use roots and indexes * wip * fix req/resp types * move blob identifier to consensus types --- .../src/rpc/codec/ssz_snappy.rs | 33 +++++++++-------- .../lighthouse_network/src/rpc/methods.rs | 37 +++++++------------ .../lighthouse_network/src/rpc/outbound.rs | 2 +- .../lighthouse_network/src/rpc/protocol.rs | 6 ++- .../src/service/api_types.rs | 9 ++--- .../lighthouse_network/src/service/mod.rs | 2 +- .../beacon_processor/worker/rpc_methods.rs | 4 +- consensus/types/src/blob_sidecar.rs | 7 ++++ 8 files changed, 51 insertions(+), 49 deletions(-) diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index d94e8f52218..a6b0deb529c 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -15,11 +15,11 @@ use std::io::{Read, Write}; use std::marker::PhantomData; use std::sync::Arc; use tokio_util::codec::{Decoder, Encoder}; -use types::light_client_bootstrap::LightClientBootstrap; +use types::{light_client_bootstrap::LightClientBootstrap, BlobSidecar}; use types::{ - BlobsSidecar, EthSpec, ForkContext, ForkName, Hash256, SignedBeaconBlock, - SignedBeaconBlockAltair, SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockBase, - SignedBeaconBlockCapella, SignedBeaconBlockEip4844, SignedBeaconBlockMerge, + EthSpec, ForkContext, ForkName, Hash256, SignedBeaconBlock, SignedBeaconBlockAltair, + SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockEip4844, + SignedBeaconBlockMerge, }; use unsigned_varint::codec::Uvi; @@ -73,7 +73,7 @@ impl Encoder> for SSZSnappyInboundCodec< RPCResponse::BlocksByRange(res) => res.as_ssz_bytes(), RPCResponse::BlocksByRoot(res) => res.as_ssz_bytes(), RPCResponse::BlobsByRange(res) => res.as_ssz_bytes(), - RPCResponse::BlockAndBlobsByRoot(res) => res.as_ssz_bytes(), + RPCResponse::SidecarByRoot(res) => res.as_ssz_bytes(), RPCResponse::LightClientBootstrap(res) => res.as_ssz_bytes(), RPCResponse::Pong(res) => res.data.as_ssz_bytes(), RPCResponse::MetaData(res) => @@ -234,7 +234,7 @@ impl Encoder> for SSZSnappyOutboundCodec< OutboundRequest::BlocksByRange(req) => req.as_ssz_bytes(), OutboundRequest::BlocksByRoot(req) => req.block_roots.as_ssz_bytes(), OutboundRequest::BlobsByRange(req) => req.as_ssz_bytes(), - OutboundRequest::BlobsByRoot(req) => req.block_roots.as_ssz_bytes(), + OutboundRequest::BlobsByRoot(req) => req.blob_ids.as_ssz_bytes(), OutboundRequest::Ping(req) => req.as_ssz_bytes(), OutboundRequest::MetaData(_) => return Ok(()), // no metadata to encode OutboundRequest::LightClientBootstrap(req) => req.as_ssz_bytes(), @@ -439,8 +439,7 @@ fn context_bytes( SignedBeaconBlock::Base { .. } => Some(fork_context.genesis_context_bytes()), }; } - if let RPCResponse::BlobsByRange(_) | RPCResponse::BlockAndBlobsByRoot(_) = rpc_variant - { + if let RPCResponse::BlobsByRange(_) | RPCResponse::SidecarByRoot(_) = rpc_variant { return fork_context.to_context_bytes(ForkName::Eip4844); } } @@ -497,7 +496,7 @@ fn handle_v1_request( BlobsByRangeRequest::from_ssz_bytes(decoded_buffer)?, ))), Protocol::BlobsByRoot => Ok(Some(InboundRequest::BlobsByRoot(BlobsByRootRequest { - block_roots: VariableList::from_ssz_bytes(decoded_buffer)?, + blob_ids: VariableList::from_ssz_bytes(decoded_buffer)?, }))), Protocol::Ping => Ok(Some(InboundRequest::Ping(Ping { data: u64::from_ssz_bytes(decoded_buffer)?, @@ -582,7 +581,7 @@ fn handle_v1_response( })?; match fork_name { ForkName::Eip4844 => Ok(Some(RPCResponse::BlobsByRange(Arc::new( - BlobsSidecar::from_ssz_bytes(decoded_buffer)?, + BlobSidecar::from_ssz_bytes(decoded_buffer)?, )))), _ => Err(RPCError::ErrorResponse( RPCResponseErrorCode::InvalidRequest, @@ -598,8 +597,8 @@ fn handle_v1_response( ) })?; match fork_name { - ForkName::Eip4844 => Ok(Some(RPCResponse::BlockAndBlobsByRoot( - SignedBeaconBlockAndBlobsSidecar::from_ssz_bytes(decoded_buffer)?, + ForkName::Eip4844 => Ok(Some(RPCResponse::SidecarByRoot( + BlobSidecar::from_ssz_bytes(decoded_buffer)?, ))), _ => Err(RPCError::ErrorResponse( RPCResponseErrorCode::InvalidRequest, @@ -738,8 +737,9 @@ mod tests { }; use std::sync::Arc; use types::{ - BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockMerge, EmptyBlock, Epoch, - ForkContext, FullPayload, Hash256, Signature, SignedBeaconBlock, Slot, + blob_sidecar::BlobIdentifier, BeaconBlock, BeaconBlockAltair, BeaconBlockBase, + BeaconBlockMerge, EmptyBlock, Epoch, ForkContext, FullPayload, Hash256, Signature, + SignedBeaconBlock, Slot, }; use snap::write::FrameEncoder; @@ -846,7 +846,10 @@ mod tests { fn blbroot_request() -> BlobsByRootRequest { BlobsByRootRequest { - block_roots: VariableList::from(vec![Hash256::zero()]), + blob_ids: VariableList::from(vec![BlobIdentifier { + block_root: Hash256::zero(), + index: 0, + }]), } } diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index 2ee4cc97987..9e386ae5166 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -12,9 +12,9 @@ use std::ops::Deref; use std::sync::Arc; use strum::IntoStaticStr; use superstruct::superstruct; -use types::SignedBeaconBlockAndBlobsSidecar; +use types::blob_sidecar::BlobIdentifier; use types::{ - blobs_sidecar::BlobsSidecar, light_client_bootstrap::LightClientBootstrap, Epoch, EthSpec, + blob_sidecar::BlobSidecar, light_client_bootstrap::LightClientBootstrap, Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot, }; @@ -26,6 +26,8 @@ pub const MAX_REQUEST_BLOCKS: u64 = 1024; pub type MaxErrorLen = U256; pub const MAX_ERROR_LEN: u64 = 256; +pub const MAX_REQUEST_BLOCKS_DENEB: u64 = 128; + // TODO: this is calculated as MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK and // MAX_BLOBS_PER_BLOCK comes from the spec. // MAX_REQUEST_BLOCKS_DENEB = 128 @@ -243,7 +245,7 @@ pub struct OldBlocksByRangeRequest { } /// Request a number of beacon block bodies from a peer. -#[derive(Clone, Debug, PartialEq)] +#[derive(Encode, Decode, Clone, Debug, PartialEq)] pub struct BlocksByRootRequest { /// The list of beacon block bodies being requested. pub block_roots: VariableList, @@ -253,14 +255,7 @@ pub struct BlocksByRootRequest { #[derive(Clone, Debug, PartialEq)] pub struct BlobsByRootRequest { /// The list of beacon block roots being requested. - pub block_roots: VariableList, -} - -impl From for BlobsByRootRequest { - fn from(r: BlocksByRootRequest) -> Self { - let BlocksByRootRequest { block_roots } = r; - Self { block_roots } - } + pub blob_ids: VariableList, } /* RPC Handling and Grouping */ @@ -279,13 +274,13 @@ pub enum RPCResponse { BlocksByRoot(Arc>), /// A response to a get BLOBS_BY_RANGE request - BlobsByRange(Arc>), + BlobsByRange(Arc>), /// A response to a get LIGHTCLIENT_BOOTSTRAP request. LightClientBootstrap(LightClientBootstrap), /// A response to a get BLOBS_BY_ROOT request. - BlockAndBlobsByRoot(SignedBeaconBlockAndBlobsSidecar), + SidecarByRoot(BlobSidecar), /// A PONG response to a PING request. Pong(Ping), @@ -378,7 +373,7 @@ impl RPCCodedResponse { RPCResponse::BlocksByRange(_) => true, RPCResponse::BlocksByRoot(_) => true, RPCResponse::BlobsByRange(_) => true, - RPCResponse::BlockAndBlobsByRoot(_) => true, + RPCResponse::SidecarByRoot(_) => true, RPCResponse::Pong(_) => false, RPCResponse::MetaData(_) => false, RPCResponse::LightClientBootstrap(_) => false, @@ -416,7 +411,7 @@ impl RPCResponse { RPCResponse::BlocksByRange(_) => Protocol::BlocksByRange, RPCResponse::BlocksByRoot(_) => Protocol::BlocksByRoot, RPCResponse::BlobsByRange(_) => Protocol::BlobsByRange, - RPCResponse::BlockAndBlobsByRoot(_) => Protocol::BlobsByRoot, + RPCResponse::SidecarByRoot(_) => Protocol::BlobsByRoot, RPCResponse::Pong(_) => Protocol::Ping, RPCResponse::MetaData(_) => Protocol::MetaData, RPCResponse::LightClientBootstrap(_) => Protocol::LightClientBootstrap, @@ -455,14 +450,10 @@ impl std::fmt::Display for RPCResponse { write!(f, "BlocksByRoot: Block slot: {}", block.slot()) } RPCResponse::BlobsByRange(blob) => { - write!(f, "BlobsByRange: Blob slot: {}", blob.beacon_block_slot) + write!(f, "BlobsByRange: Blob slot: {}", blob.slot) } - RPCResponse::BlockAndBlobsByRoot(blob) => { - write!( - f, - "BlobsByRoot: Blob slot: {}", - blob.blobs_sidecar.beacon_block_slot - ) + RPCResponse::SidecarByRoot(sidecar) => { + write!(f, "BlobsByRoot: Blob slot: {}", sidecar.slot) } RPCResponse::Pong(ping) => write!(f, "Pong: {}", ping.data), RPCResponse::MetaData(metadata) => write!(f, "Metadata: {}", metadata.seq_number()), @@ -520,7 +511,7 @@ impl std::fmt::Display for BlobsByRootRequest { write!( f, "Request: BlobsByRoot: Number of Requested Roots: {}", - self.block_roots.len() + self.blob_ids.len() ) } } diff --git a/beacon_node/lighthouse_network/src/rpc/outbound.rs b/beacon_node/lighthouse_network/src/rpc/outbound.rs index 090db77eca9..6115f874efb 100644 --- a/beacon_node/lighthouse_network/src/rpc/outbound.rs +++ b/beacon_node/lighthouse_network/src/rpc/outbound.rs @@ -113,7 +113,7 @@ impl OutboundRequest { OutboundRequest::BlocksByRange(req) => req.count, OutboundRequest::BlocksByRoot(req) => req.block_roots.len() as u64, OutboundRequest::BlobsByRange(req) => req.count, - OutboundRequest::BlobsByRoot(req) => req.block_roots.len() as u64, + OutboundRequest::BlobsByRoot(req) => req.blob_ids.len() as u64, OutboundRequest::Ping(_) => 1, OutboundRequest::MetaData(_) => 1, OutboundRequest::LightClientBootstrap(_) => 1, diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index f002000a513..bf7d9ca2a88 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -194,7 +194,7 @@ pub enum Protocol { #[strum(serialize = "blobs_sidecars_by_range")] BlobsByRange, /// The `BlobsByRoot` protocol name. - #[strum(serialize = "beacon_block_and_blobs_sidecar_by_root")] + #[strum(serialize = "blob_sidecars_by_root")] BlobsByRoot, /// The `Ping` protocol name. Ping, @@ -360,6 +360,7 @@ impl ProtocolId { ::ssz_fixed_len(), ), Protocol::BlobsByRoot => { + // TODO: This looks wrong to me RpcLimits::new(*BLOCKS_BY_ROOT_REQUEST_MIN, *BLOCKS_BY_ROOT_REQUEST_MAX) } Protocol::Ping => RpcLimits::new( @@ -386,6 +387,7 @@ impl ProtocolId { Protocol::BlocksByRoot => rpc_block_limits_by_fork(fork_context.current_fork()), Protocol::BlobsByRange => RpcLimits::new(*BLOBS_SIDECAR_MIN, *BLOBS_SIDECAR_MAX), Protocol::BlobsByRoot => { + // TODO: wrong too RpcLimits::new(*SIGNED_BLOCK_AND_BLOBS_MIN, *SIGNED_BLOCK_AND_BLOBS_MAX) } Protocol::Ping => RpcLimits::new( @@ -524,7 +526,7 @@ impl InboundRequest { InboundRequest::BlocksByRange(req) => req.count, InboundRequest::BlocksByRoot(req) => req.block_roots.len() as u64, InboundRequest::BlobsByRange(req) => req.count, - InboundRequest::BlobsByRoot(req) => req.block_roots.len() as u64, + InboundRequest::BlobsByRoot(req) => req.blob_ids.len() as u64, InboundRequest::Ping(_) => 1, InboundRequest::MetaData(_) => 1, InboundRequest::LightClientBootstrap(_) => 1, diff --git a/beacon_node/lighthouse_network/src/service/api_types.rs b/beacon_node/lighthouse_network/src/service/api_types.rs index c9c239d8cf4..4a1d125fa4f 100644 --- a/beacon_node/lighthouse_network/src/service/api_types.rs +++ b/beacon_node/lighthouse_network/src/service/api_types.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use libp2p::core::connection::ConnectionId; use types::light_client_bootstrap::LightClientBootstrap; -use types::{BlobsSidecar, EthSpec, SignedBeaconBlock}; +use types::{BlobSidecar, EthSpec, SignedBeaconBlock}; use crate::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest}; use crate::rpc::{ @@ -12,7 +12,6 @@ use crate::rpc::{ }, OutboundRequest, SubstreamId, }; -use types::SignedBeaconBlockAndBlobsSidecar; /// Identifier of requests sent by a peer. pub type PeerRequestId = (ConnectionId, SubstreamId); @@ -77,13 +76,13 @@ pub enum Response { /// A response to a get BLOCKS_BY_RANGE request. A None response signals the end of the batch. BlocksByRange(Option>>), /// A response to a get BLOBS_BY_RANGE request. A None response signals the end of the batch. - BlobsByRange(Option>>), + BlobsByRange(Option>>), /// A response to a get BLOCKS_BY_ROOT request. BlocksByRoot(Option>>), /// A response to a LightClientUpdate request. LightClientBootstrap(LightClientBootstrap), /// A response to a get BLOBS_BY_ROOT request. - BlobsByRoot(Option>), + BlobsByRoot(Option>), } impl std::convert::From> for RPCCodedResponse { @@ -98,7 +97,7 @@ impl std::convert::From> for RPCCodedResponse RPCCodedResponse::StreamTermination(ResponseTermination::BlocksByRange), }, Response::BlobsByRoot(r) => match r { - Some(b) => RPCCodedResponse::Success(RPCResponse::BlockAndBlobsByRoot(b)), + Some(b) => RPCCodedResponse::Success(RPCResponse::SidecarByRoot(b)), None => RPCCodedResponse::StreamTermination(ResponseTermination::BlobsByRoot), }, Response::BlobsByRange(r) => match r { diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 9e45c6ad3e5..4c8c770b945 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1328,7 +1328,7 @@ impl Network { RPCResponse::BlocksByRoot(resp) => { self.build_response(id, peer_id, Response::BlocksByRoot(Some(resp))) } - RPCResponse::BlockAndBlobsByRoot(resp) => { + RPCResponse::SidecarByRoot(resp) => { self.build_response(id, peer_id, Response::BlobsByRoot(Some(resp))) } // Should never be reached diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index ad0595ea05b..85781de1f12 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -224,7 +224,7 @@ impl Worker { async move { let mut send_block_count = 0; let mut send_response = true; - for root in request.block_roots.iter() { + for root in request.blob_ids.iter() { match self .chain .get_block_and_blobs_checking_early_attester_cache(root) @@ -329,7 +329,7 @@ impl Worker { self.log, "Received BlobsByRoot Request"; "peer" => %peer_id, - "requested" => request.block_roots.len(), + "requested" => request.blob_ids.len(), "returned" => send_block_count ); diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 3a2012839e6..12a6633dee6 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -8,6 +8,13 @@ use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; +/// Container of the data that identifies an individual blob. +#[derive(Encode, Decode, Clone, Debug, PartialEq)] +pub struct BlobIdentifier { + pub block_root: Hash256, + pub index: u64, +} + #[derive( Debug, Clone, From 3898cf7be8ae5eb79dc19027cad54060024cd0d9 Mon Sep 17 00:00:00 2001 From: Divma <26765164+divagant-martian@users.noreply.github.com> Date: Fri, 10 Mar 2023 05:53:36 -0500 Subject: [PATCH 06/96] Modify `lighthouse_network` gossip types to free the blobs (#4064) * Modify blob topics * add signedblol type pubsun messages are signed * improve subnet topic index * improve display code * fix parse code --------- Co-authored-by: Pawan Dhananjay --- .../src/service/gossip_cache.rs | 10 ++--- .../lighthouse_network/src/service/mod.rs | 1 + .../lighthouse_network/src/service/utils.rs | 5 ++- .../lighthouse_network/src/types/pubsub.rs | 35 +++++++-------- .../lighthouse_network/src/types/topics.rs | 45 +++++++++---------- consensus/types/src/lib.rs | 2 + consensus/types/src/signed_blob.rs | 24 ++++++++++ 7 files changed, 74 insertions(+), 48 deletions(-) create mode 100644 consensus/types/src/signed_blob.rs diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index d3971a7d742..a6e003934cd 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -21,7 +21,7 @@ pub struct GossipCache { /// Timeout for blocks. beacon_block: Option, /// Timeout for blobs. - beacon_block_and_blobs_sidecar: Option, + blob_sidecar: Option, /// Timeout for aggregate attestations. aggregates: Option, /// Timeout for attestations. @@ -50,7 +50,7 @@ pub struct GossipCacheBuilder { /// Timeout for blocks. beacon_block: Option, /// Timeout for blob sidecars. - beacon_block_and_blobs_sidecar: Option, + blob_sidecar: Option, /// Timeout for aggregate attestations. aggregates: Option, /// Timeout for attestations. @@ -151,7 +151,7 @@ impl GossipCacheBuilder { let GossipCacheBuilder { default_timeout, beacon_block, - beacon_block_and_blobs_sidecar, + blob_sidecar, aggregates, attestation, voluntary_exit, @@ -167,7 +167,7 @@ impl GossipCacheBuilder { expirations: DelayQueue::default(), topic_msgs: HashMap::default(), beacon_block: beacon_block.or(default_timeout), - beacon_block_and_blobs_sidecar: beacon_block_and_blobs_sidecar.or(default_timeout), + blob_sidecar: blob_sidecar.or(default_timeout), aggregates: aggregates.or(default_timeout), attestation: attestation.or(default_timeout), voluntary_exit: voluntary_exit.or(default_timeout), @@ -193,7 +193,7 @@ impl GossipCache { pub fn insert(&mut self, topic: GossipTopic, data: Vec) { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, - GossipKind::BeaconBlocksAndBlobsSidecar => self.beacon_block_and_blobs_sidecar, + GossipKind::BlobSidecar(_) => self.blob_sidecar, GossipKind::BeaconAggregateAndProof => self.aggregates, GossipKind::Attestation(_) => self.attestation, GossipKind::VoluntaryExit => self.voluntary_exit, diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 4c8c770b945..0154a4dd328 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -235,6 +235,7 @@ impl Network { possible_fork_digests, ctx.chain_spec.attestation_subnet_count, SYNC_COMMITTEE_SUBNET_COUNT, + 4, // TODO(pawan): get this from chainspec ), max_subscribed_topics: 200, max_subscriptions_per_request: 150, // 148 in theory = (64 attestation + 4 sync committee + 6 core topics) * 2 diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index 383b78abf24..e0eb3ff50db 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -236,6 +236,7 @@ pub(crate) fn create_whitelist_filter( possible_fork_digests: Vec<[u8; 4]>, attestation_subnet_count: u64, sync_committee_subnet_count: u64, + blob_sidecar_subnet_count: u64, ) -> WhitelistSubscriptionFilter { let mut possible_hashes = HashSet::new(); for fork_digest in possible_fork_digests { @@ -255,13 +256,15 @@ pub(crate) fn create_whitelist_filter( add(BlsToExecutionChange); add(LightClientFinalityUpdate); add(LightClientOptimisticUpdate); - add(BeaconBlocksAndBlobsSidecar); for id in 0..attestation_subnet_count { add(Attestation(SubnetId::new(id))); } for id in 0..sync_committee_subnet_count { add(SyncCommitteeMessage(SyncSubnetId::new(id))); } + for id in 0..blob_sidecar_subnet_count { + add(BlobSidecar(id)); + } } WhitelistSubscriptionFilter(possible_hashes) } diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 7951a072438..243e8314850 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -11,8 +11,8 @@ use std::sync::Arc; use types::{ Attestation, AttesterSlashing, EthSpec, ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, - SignedBeaconBlockAltair, SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockBase, - SignedBeaconBlockCapella, SignedBeaconBlockMerge, SignedBlsToExecutionChange, + SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, + SignedBeaconBlockMerge, SignedBlobSidecar, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; @@ -20,8 +20,8 @@ use types::{ pub enum PubsubMessage { /// Gossipsub message providing notification of a new block. BeaconBlock(Arc>), - /// Gossipsub message providing notification of a new SignedBeaconBlock coupled with a blobs sidecar. - BeaconBlockAndBlobsSidecars(SignedBeaconBlockAndBlobsSidecar), + /// Gossipsub message providing notification of a [`SignedBlobSidecar`] along with the subnet id where it was received. + BlobSidecar(Box<(u64, SignedBlobSidecar)>), /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), /// Gossipsub message providing notification of a raw un-aggregated attestation with its shard id. @@ -115,8 +115,8 @@ impl PubsubMessage { pub fn kind(&self) -> GossipKind { match self { PubsubMessage::BeaconBlock(_) => GossipKind::BeaconBlock, - PubsubMessage::BeaconBlockAndBlobsSidecars(_) => { - GossipKind::BeaconBlocksAndBlobsSidecar + PubsubMessage::BlobSidecar(blob_sidecar_data) => { + GossipKind::BlobSidecar(blob_sidecar_data.0) } PubsubMessage::AggregateAndProofAttestation(_) => GossipKind::BeaconAggregateAndProof, PubsubMessage::Attestation(attestation_data) => { @@ -203,15 +203,15 @@ impl PubsubMessage { }; Ok(PubsubMessage::BeaconBlock(Arc::new(beacon_block))) } - GossipKind::BeaconBlocksAndBlobsSidecar => { + GossipKind::BlobSidecar(blob_index) => { match fork_context.from_context_bytes(gossip_topic.fork_digest) { Some(ForkName::Eip4844) => { - let block_and_blobs_sidecar = - SignedBeaconBlockAndBlobsSidecar::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?; - Ok(PubsubMessage::BeaconBlockAndBlobsSidecars( - block_and_blobs_sidecar, - )) + let blob_sidecar = SignedBlobSidecar::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?; + Ok(PubsubMessage::BlobSidecar(Box::new(( + *blob_index, + blob_sidecar, + )))) } Some( ForkName::Base @@ -293,7 +293,7 @@ impl PubsubMessage { // messages for us. match &self { PubsubMessage::BeaconBlock(data) => data.as_ssz_bytes(), - PubsubMessage::BeaconBlockAndBlobsSidecars(data) => data.as_ssz_bytes(), + PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), PubsubMessage::ProposerSlashing(data) => data.as_ssz_bytes(), @@ -317,11 +317,10 @@ impl std::fmt::Display for PubsubMessage { block.slot(), block.message().proposer_index() ), - PubsubMessage::BeaconBlockAndBlobsSidecars(block_and_blob) => write!( + PubsubMessage::BlobSidecar(data) => write!( f, - "Beacon block and Blobs Sidecar: slot: {}, blobs: {}", - block_and_blob.beacon_block.message().slot(), - block_and_blob.blobs_sidecar.blobs.len(), + "BlobSidecar: slot: {}, blob index: {}", + data.1.blob.slot, data.1.blob.index, ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 20f836b76a1..88764fb9514 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -11,9 +11,9 @@ use crate::Subnet; pub const TOPIC_PREFIX: &str = "eth2"; pub const SSZ_SNAPPY_ENCODING_POSTFIX: &str = "ssz_snappy"; pub const BEACON_BLOCK_TOPIC: &str = "beacon_block"; -pub const BEACON_BLOCK_AND_BLOBS_SIDECAR_TOPIC: &str = "beacon_block_and_blobs_sidecar"; pub const BEACON_AGGREGATE_AND_PROOF_TOPIC: &str = "beacon_aggregate_and_proof"; pub const BEACON_ATTESTATION_PREFIX: &str = "beacon_attestation_"; +pub const BLOB_SIDECAR_PREFIX: &str = "blob_sidecar_"; pub const VOLUNTARY_EXIT_TOPIC: &str = "voluntary_exit"; pub const PROPOSER_SLASHING_TOPIC: &str = "proposer_slashing"; pub const ATTESTER_SLASHING_TOPIC: &str = "attester_slashing"; @@ -82,10 +82,10 @@ pub struct GossipTopic { pub enum GossipKind { /// Topic for publishing beacon blocks. BeaconBlock, - /// Topic for publishing beacon block coupled with blob sidecars. - BeaconBlocksAndBlobsSidecar, /// Topic for publishing aggregate attestations and proofs. BeaconAggregateAndProof, + /// Topic for publishing BlobSidecars. + BlobSidecar(u64), /// Topic for publishing raw attestations on a particular subnet. #[strum(serialize = "beacon_attestation")] Attestation(SubnetId), @@ -115,6 +115,9 @@ impl std::fmt::Display for GossipKind { GossipKind::SyncCommitteeMessage(subnet_id) => { write!(f, "sync_committee_{}", **subnet_id) } + GossipKind::BlobSidecar(blob_index) => { + write!(f, "{}{}", BLOB_SIDECAR_PREFIX, blob_index) + } x => f.write_str(x.as_ref()), } } @@ -175,7 +178,6 @@ impl GossipTopic { let kind = match topic_parts[3] { BEACON_BLOCK_TOPIC => GossipKind::BeaconBlock, BEACON_AGGREGATE_AND_PROOF_TOPIC => GossipKind::BeaconAggregateAndProof, - BEACON_BLOCK_AND_BLOBS_SIDECAR_TOPIC => GossipKind::BeaconBlocksAndBlobsSidecar, SIGNED_CONTRIBUTION_AND_PROOF_TOPIC => GossipKind::SignedContributionAndProof, VOLUNTARY_EXIT_TOPIC => GossipKind::VoluntaryExit, PROPOSER_SLASHING_TOPIC => GossipKind::ProposerSlashing, @@ -183,11 +185,8 @@ impl GossipTopic { BLS_TO_EXECUTION_CHANGE_TOPIC => GossipKind::BlsToExecutionChange, LIGHT_CLIENT_FINALITY_UPDATE => GossipKind::LightClientFinalityUpdate, LIGHT_CLIENT_OPTIMISTIC_UPDATE => GossipKind::LightClientOptimisticUpdate, - topic => match committee_topic_index(topic) { - Some(subnet) => match subnet { - Subnet::Attestation(s) => GossipKind::Attestation(s), - Subnet::SyncCommittee(s) => GossipKind::SyncCommitteeMessage(s), - }, + topic => match subnet_topic_index(topic) { + Some(kind) => kind, None => return Err(format!("Unknown topic: {}", topic)), }, }; @@ -232,7 +231,6 @@ impl std::fmt::Display for GossipTopic { let kind = match self.kind { GossipKind::BeaconBlock => BEACON_BLOCK_TOPIC.into(), - GossipKind::BeaconBlocksAndBlobsSidecar => BEACON_BLOCK_AND_BLOBS_SIDECAR_TOPIC.into(), GossipKind::BeaconAggregateAndProof => BEACON_AGGREGATE_AND_PROOF_TOPIC.into(), GossipKind::VoluntaryExit => VOLUNTARY_EXIT_TOPIC.into(), GossipKind::ProposerSlashing => PROPOSER_SLASHING_TOPIC.into(), @@ -242,6 +240,9 @@ impl std::fmt::Display for GossipTopic { GossipKind::SyncCommitteeMessage(index) => { format!("{}{}", SYNC_COMMITTEE_PREFIX_TOPIC, *index) } + GossipKind::BlobSidecar(blob_index) => { + format!("{}{}", BLOB_SIDECAR_PREFIX, blob_index) + } GossipKind::BlsToExecutionChange => BLS_TO_EXECUTION_CHANGE_TOPIC.into(), GossipKind::LightClientFinalityUpdate => LIGHT_CLIENT_FINALITY_UPDATE.into(), GossipKind::LightClientOptimisticUpdate => LIGHT_CLIENT_OPTIMISTIC_UPDATE.into(), @@ -273,22 +274,18 @@ pub fn subnet_from_topic_hash(topic_hash: &TopicHash) -> Option { GossipTopic::decode(topic_hash.as_str()).ok()?.subnet_id() } -// Determines if a string is an attestation or sync committee topic. -fn committee_topic_index(topic: &str) -> Option { - if topic.starts_with(BEACON_ATTESTATION_PREFIX) { - return Some(Subnet::Attestation(SubnetId::new( - topic - .trim_start_matches(BEACON_ATTESTATION_PREFIX) - .parse::() - .ok()?, +// Determines if the topic name is of an indexed topic. +fn subnet_topic_index(topic: &str) -> Option { + if let Some(index) = topic.strip_prefix(BEACON_ATTESTATION_PREFIX) { + return Some(GossipKind::Attestation(SubnetId::new( + index.parse::().ok()?, ))); - } else if topic.starts_with(SYNC_COMMITTEE_PREFIX_TOPIC) { - return Some(Subnet::SyncCommittee(SyncSubnetId::new( - topic - .trim_start_matches(SYNC_COMMITTEE_PREFIX_TOPIC) - .parse::() - .ok()?, + } else if let Some(index) = topic.strip_prefix(SYNC_COMMITTEE_PREFIX_TOPIC) { + return Some(GossipKind::SyncCommitteeMessage(SyncSubnetId::new( + index.parse::().ok()?, ))); + } else if let Some(index) = topic.strip_prefix(BLOB_SIDECAR_PREFIX) { + return Some(GossipKind::BlobSidecar(index.parse::().ok()?)); } None } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index a2dfb853ead..ce483785322 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -101,6 +101,7 @@ pub mod sqlite; pub mod blob_sidecar; pub mod blobs_sidecar; +pub mod signed_blob; pub mod signed_block_and_blobs; pub mod transaction; @@ -181,6 +182,7 @@ pub use crate::signed_beacon_block::{ SignedBlindedBeaconBlock, }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; +pub use crate::signed_blob::*; pub use crate::signed_block_and_blobs::SignedBeaconBlockAndBlobsSidecar; pub use crate::signed_block_and_blobs::SignedBeaconBlockAndBlobsSidecarDecode; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs new file mode 100644 index 00000000000..a3cfb9d58af --- /dev/null +++ b/consensus/types/src/signed_blob.rs @@ -0,0 +1,24 @@ +use crate::{test_utils::TestRandom, BlobSidecar, EthSpec, Signature}; +use serde_derive::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + Debug, + Clone, + PartialEq, + Serialize, + Deserialize, + Encode, + Decode, + TestRandom, + TreeHash, + arbitrary::Arbitrary, +)] +#[serde(bound = "T: EthSpec")] +#[arbitrary(bound = "T: EthSpec")] +pub struct SignedBlobSidecar { + pub blob: BlobSidecar, + pub signature: Signature, +} From 140bdd370d3f16882174c14618539138df1cf49b Mon Sep 17 00:00:00 2001 From: Divma <26765164+divagant-martian@users.noreply.github.com> Date: Fri, 10 Mar 2023 06:22:31 -0500 Subject: [PATCH 07/96] update code paths in the `network` crate (#4065) * wip * fix router * arc the byroot responses we send * add placeholder for blob verification * respond to blobs by range and blobs by root request in the most horrible and gross way ever * everything in sync is now unimplemented * fix compiation issues * http_pi change is very small, just add it * remove ctrl-c ctrl-v's docs --- beacon_node/http_api/src/publish_blocks.rs | 6 +- .../src/rpc/codec/ssz_snappy.rs | 4 +- .../lighthouse_network/src/rpc/methods.rs | 2 +- .../src/service/api_types.rs | 2 +- .../network/src/beacon_processor/mod.rs | 33 +++++---- .../beacon_processor/worker/gossip_methods.rs | 24 +++++- .../beacon_processor/worker/rpc_methods.rs | 73 +++++++++++++++---- beacon_node/network/src/router/mod.rs | 16 ++-- beacon_node/network/src/router/processor.rs | 30 ++++---- beacon_node/network/src/sync/manager.rs | 64 ++-------------- .../network/src/sync/network_context.rs | 6 +- 11 files changed, 138 insertions(+), 122 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 346e802cacb..32494367363 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -44,11 +44,7 @@ pub async fn publish_block( beacon_block: block, blobs_sidecar: Arc::new(sidecar), }; - crate::publish_pubsub_message( - network_tx, - PubsubMessage::BeaconBlockAndBlobsSidecars(block_and_blobs.clone()), - )?; - block_and_blobs.into() + unimplemented!("Needs to be adjusted") } else { //FIXME(sean): This should probably return a specific no-blob-cached error code, beacon API coordination required return Err(warp_utils::reject::broadcast_without_import( diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index a6b0deb529c..ab61f45be94 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -597,9 +597,9 @@ fn handle_v1_response( ) })?; match fork_name { - ForkName::Eip4844 => Ok(Some(RPCResponse::SidecarByRoot( + ForkName::Eip4844 => Ok(Some(RPCResponse::SidecarByRoot(Arc::new( BlobSidecar::from_ssz_bytes(decoded_buffer)?, - ))), + )))), _ => Err(RPCError::ErrorResponse( RPCResponseErrorCode::InvalidRequest, "Invalid fork name for block and blobs by root".to_string(), diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index 9e386ae5166..63a3fc6a196 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -280,7 +280,7 @@ pub enum RPCResponse { LightClientBootstrap(LightClientBootstrap), /// A response to a get BLOBS_BY_ROOT request. - SidecarByRoot(BlobSidecar), + SidecarByRoot(Arc>), /// A PONG response to a PING request. Pong(Ping), diff --git a/beacon_node/lighthouse_network/src/service/api_types.rs b/beacon_node/lighthouse_network/src/service/api_types.rs index 4a1d125fa4f..fc65959d1c6 100644 --- a/beacon_node/lighthouse_network/src/service/api_types.rs +++ b/beacon_node/lighthouse_network/src/service/api_types.rs @@ -82,7 +82,7 @@ pub enum Response { /// A response to a LightClientUpdate request. LightClientBootstrap(LightClientBootstrap), /// A response to a get BLOBS_BY_ROOT request. - BlobsByRoot(Option>), + BlobsByRoot(Option>>), } impl std::convert::From> for RPCCodedResponse { diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/network/src/beacon_processor/mod.rs index 73535dc83c9..410389b1195 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/network/src/beacon_processor/mod.rs @@ -65,7 +65,7 @@ use task_executor::TaskExecutor; use tokio::sync::mpsc; use types::{ Attestation, AttesterSlashing, Hash256, LightClientFinalityUpdate, LightClientOptimisticUpdate, - ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar, + ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecar, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; @@ -444,20 +444,22 @@ impl WorkEvent { } /// Create a new `Work` event for some blobs sidecar. - pub fn gossip_block_and_blobs_sidecar( + pub fn gossip_signed_blob_sidecar( message_id: MessageId, peer_id: PeerId, peer_client: Client, - block_and_blobs: SignedBeaconBlockAndBlobsSidecar, + blob_index: u64, + signed_blob: Arc>, seen_timestamp: Duration, ) -> Self { Self { drop_during_sync: false, - work: Work::GossipBlockAndBlobsSidecar { + work: Work::GossipSignedBlobSidecar { message_id, peer_id, peer_client, - block_and_blobs, + blob_index, + signed_blob, seen_timestamp, }, } @@ -857,11 +859,12 @@ pub enum Work { block: Arc>, seen_timestamp: Duration, }, - GossipBlockAndBlobsSidecar { + GossipSignedBlobSidecar { message_id: MessageId, peer_id: PeerId, peer_client: Client, - block_and_blobs: SignedBeaconBlockAndBlobsSidecar, + blob_index: u64, + signed_blob: Arc>, seen_timestamp: Duration, }, DelayedImportBlock { @@ -965,7 +968,7 @@ impl Work { Work::GossipAggregate { .. } => GOSSIP_AGGREGATE, Work::GossipAggregateBatch { .. } => GOSSIP_AGGREGATE_BATCH, Work::GossipBlock { .. } => GOSSIP_BLOCK, - Work::GossipBlockAndBlobsSidecar { .. } => GOSSIP_BLOCK_AND_BLOBS_SIDECAR, + Work::GossipSignedBlobSidecar { .. } => GOSSIP_BLOCK_AND_BLOBS_SIDECAR, Work::DelayedImportBlock { .. } => DELAYED_IMPORT_BLOCK, Work::GossipVoluntaryExit { .. } => GOSSIP_VOLUNTARY_EXIT, Work::GossipProposerSlashing { .. } => GOSSIP_PROPOSER_SLASHING, @@ -1459,7 +1462,7 @@ impl BeaconProcessor { Work::GossipBlock { .. } => { gossip_block_queue.push(work, work_id, &self.log) } - Work::GossipBlockAndBlobsSidecar { .. } => { + Work::GossipSignedBlobSidecar { .. } => { gossip_block_and_blobs_sidecar_queue.push(work, work_id, &self.log) } Work::DelayedImportBlock { .. } => { @@ -1742,21 +1745,21 @@ impl BeaconProcessor { /* * Verification for blobs sidecars received on gossip. */ - Work::GossipBlockAndBlobsSidecar { + Work::GossipSignedBlobSidecar { message_id, peer_id, peer_client, - block_and_blobs: block_sidecar_pair, + blob_index, + signed_blob, seen_timestamp, } => task_spawner.spawn_async(async move { worker - .process_gossip_block( + .process_gossip_blob( message_id, peer_id, peer_client, - block_sidecar_pair.into(), - work_reprocessing_tx, - duplicate_cache, + blob_index, + signed_blob, seen_timestamp, ) .await diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index 5e84cbb5f22..0bdebd88fef 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -17,12 +17,13 @@ use operation_pool::ReceivedPreCapella; use slog::{crit, debug, error, info, trace, warn}; use slot_clock::SlotClock; use ssz::Encode; +use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tokio::sync::mpsc; use types::{ Attestation, AttesterSlashing, EthSpec, Hash256, IndexedAttestation, LightClientFinalityUpdate, - LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, + LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBlobSidecar, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; @@ -647,6 +648,27 @@ impl Worker { } } + // TODO: docs + #[allow(clippy::too_many_arguments)] + pub async fn process_gossip_blob( + self, + message_id: MessageId, + peer_id: PeerId, + peer_client: Client, + blob_index: u64, + signed_blob: Arc>, + seen_duration: Duration, + ) { + // TODO: gossip verification + crit!(self.log, "UNIMPLEMENTED gossip blob verification"; + "peer_id" => %peer_id, + "client" => %peer_client, + "blob_topic" => blob_index, + "blob_index" => signed_blob.blob.index, + "blob_slot" => signed_blob.blob.slot + ); + } + /// Process the beacon block received from the gossip network and: /// /// - If it passes gossip propagation criteria, tell the network thread to forward it. diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 85781de1f12..3fb8feaf488 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -14,6 +14,7 @@ use slog::{debug, error, warn}; use slot_clock::SlotClock; use std::sync::Arc; use task_executor::TaskExecutor; +use types::blob_sidecar::BlobIdentifier; use types::light_client_bootstrap::LightClientBootstrap; use types::{Epoch, EthSpec, Hash256, Slot}; @@ -219,24 +220,49 @@ impl Worker { request_id: PeerRequestId, request: BlobsByRootRequest, ) { + // TODO: this code is grossly adjusted to free the blobs. Needs love <3 // Fetching blocks is async because it may have to hit the execution layer for payloads. executor.spawn( async move { + let requested_blobs = request.blob_ids.len(); let mut send_block_count = 0; let mut send_response = true; - for root in request.blob_ids.iter() { + for BlobIdentifier{ block_root: root, index } in request.blob_ids.into_iter() { match self .chain - .get_block_and_blobs_checking_early_attester_cache(root) + .get_block_and_blobs_checking_early_attester_cache(&root) .await { Ok(Some(block_and_blobs)) => { - self.send_response( - peer_id, - Response::BlobsByRoot(Some(block_and_blobs)), - request_id, - ); - send_block_count += 1; + // + // TODO: HORRIBLE NSFW CODE AHEAD + // + let types::SignedBeaconBlockAndBlobsSidecar {beacon_block, blobs_sidecar} = block_and_blobs; + let types::BlobsSidecar{ beacon_block_root, beacon_block_slot, blobs: blob_bundle, kzg_aggregated_proof }: types::BlobsSidecar<_> = blobs_sidecar.as_ref().clone(); + // TODO: this should be unreachable after this is addressed seriously, + // so for now let's be ok with a panic in the expect. + let block = beacon_block.message_eip4844().expect("We fucked up the block blob stuff"); + // Intentionally not accessing the list directly + for (known_index, blob) in blob_bundle.into_iter().enumerate() { + if (known_index as u64) == index { + let blob_sidecar = types::BlobSidecar{ + block_root: beacon_block_root, + index, + slot: beacon_block_slot, + block_parent_root: block.parent_root, + proposer_index: block.proposer_index, + blob, + kzg_commitment: block.body.blob_kzg_commitments[known_index].clone(), // TODO: needs to be stored in a more logical way so that this won't panic. + kzg_proof: kzg_aggregated_proof // TODO: yeah + }; + self.send_response( + peer_id, + Response::BlobsByRoot(Some(Arc::new(blob_sidecar))), + request_id, + ); + send_block_count += 1; + } + } } Ok(None) => { debug!( @@ -250,7 +276,6 @@ impl Worker { error!( self.log, "No blobs in the store for block root"; - "request" => ?request, "peer" => %peer_id, "block_root" => ?root ); @@ -329,7 +354,7 @@ impl Worker { self.log, "Received BlobsByRoot Request"; "peer" => %peer_id, - "requested" => request.blob_ids.len(), + "requested" => requested_blobs, "returned" => send_block_count ); @@ -813,12 +838,28 @@ impl Worker { for root in block_roots { match self.chain.get_blobs(&root, data_availability_boundary) { Ok(Some(blobs)) => { - blobs_sent += 1; - self.send_network_message(NetworkMessage::SendResponse { - peer_id, - response: Response::BlobsByRange(Some(Arc::new(blobs))), - id: request_id, - }); + // TODO: more GROSS code ahead. Reader beware + let types::BlobsSidecar{ beacon_block_root, beacon_block_slot, blobs: blob_bundle, kzg_aggregated_proof }: types::BlobsSidecar<_> = blobs; + + for (blob_index, blob) in blob_bundle.into_iter().enumerate() { + let blob_sidecar = types::BlobSidecar{ + block_root: beacon_block_root, + index: blob_index as u64, + slot: beacon_block_slot, + block_parent_root: Hash256::zero(), + proposer_index: 0, + blob, + kzg_commitment: types::KzgCommitment::default(), + kzg_proof: types::KzgProof::default(), + }; + + blobs_sent += 1; + self.send_network_message(NetworkMessage::SendResponse { + peer_id, + response: Response::BlobsByRange(Some(Arc::new(blob_sidecar))), + id: request_id, + }); + } } Ok(None) => { error!( diff --git a/beacon_node/network/src/router/mod.rs b/beacon_node/network/src/router/mod.rs index 31f2092049f..a18ce4e7c81 100644 --- a/beacon_node/network/src/router/mod.rs +++ b/beacon_node/network/src/router/mod.rs @@ -204,13 +204,13 @@ impl Router { self.processor .on_blocks_by_root_response(peer_id, request_id, beacon_block); } - Response::BlobsByRange(beacon_blob) => { + Response::BlobsByRange(blob) => { self.processor - .on_blobs_by_range_response(peer_id, request_id, beacon_blob); + .on_blobs_by_range_response(peer_id, request_id, blob); } - Response::BlobsByRoot(beacon_blob) => { + Response::BlobsByRoot(blob) => { self.processor - .on_blobs_by_root_response(peer_id, request_id, beacon_blob); + .on_blobs_by_root_response(peer_id, request_id, blob); } Response::LightClientBootstrap(_) => unreachable!(), } @@ -250,12 +250,14 @@ impl Router { block, ); } - PubsubMessage::BeaconBlockAndBlobsSidecars(block_and_blobs) => { - self.processor.on_block_and_blobs_sidecar_gossip( + PubsubMessage::BlobSidecar(data) => { + let (blob_index, signed_blob) = *data; + self.processor.on_blob_sidecar_gossip( id, peer_id, self.network_globals.client(&peer_id), - block_and_blobs, + blob_index, + Arc::new(signed_blob), ); } PubsubMessage::VoluntaryExit(exit) => { diff --git a/beacon_node/network/src/router/processor.rs b/beacon_node/network/src/router/processor.rs index d0879babacb..56a5f245867 100644 --- a/beacon_node/network/src/router/processor.rs +++ b/beacon_node/network/src/router/processor.rs @@ -18,10 +18,10 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use store::SyncCommitteeMessage; use tokio::sync::mpsc; use types::{ - Attestation, AttesterSlashing, BlobsSidecar, EthSpec, LightClientFinalityUpdate, + Attestation, AttesterSlashing, BlobSidecar, EthSpec, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, - SignedBeaconBlockAndBlobsSidecar, SignedBlsToExecutionChange, SignedContributionAndProof, - SignedVoluntaryExit, SubnetId, SyncSubnetId, + SignedBlobSidecar, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, + SubnetId, SyncSubnetId, }; /// Processes validated messages from the network. It relays necessary data to the syncing thread @@ -249,7 +249,7 @@ impl Processor { &mut self, peer_id: PeerId, request_id: RequestId, - blob_sidecar: Option>>, + blob_sidecar: Option>>, ) { trace!( self.log, @@ -310,7 +310,7 @@ impl Processor { &mut self, peer_id: PeerId, request_id: RequestId, - block_and_blobs: Option>, + blob_sidecar: Option>>, ) { let request_id = match request_id { RequestId::Sync(sync_id) => match sync_id { @@ -322,18 +322,18 @@ impl Processor { unreachable!("Batch syncing does not request BBRoot requests") } }, - RequestId::Router => unreachable!("All BBRoot requests belong to sync"), + RequestId::Router => unreachable!("All BlobsByRoot requests belong to sync"), }; trace!( self.log, - "Received BlockAndBlobssByRoot Response"; + "Received BlobsByRoot Response"; "peer" => %peer_id, ); - self.send_to_sync(SyncMessage::RpcBlockAndBlobs { - peer_id, + self.send_to_sync(SyncMessage::RpcBlobs { request_id, - block_and_blobs, + peer_id, + blob_sidecar, seen_timestamp: timestamp_now(), }); } @@ -359,18 +359,20 @@ impl Processor { )) } - pub fn on_block_and_blobs_sidecar_gossip( + pub fn on_blob_sidecar_gossip( &mut self, message_id: MessageId, peer_id: PeerId, peer_client: Client, - block_and_blobs: SignedBeaconBlockAndBlobsSidecar, + blob_index: u64, // TODO: add a type for the blob index + signed_blob: Arc>, ) { - self.send_beacon_processor_work(BeaconWorkEvent::gossip_block_and_blobs_sidecar( + self.send_beacon_processor_work(BeaconWorkEvent::gossip_signed_blob_sidecar( message_id, peer_id, peer_client, - block_and_blobs, + blob_index, + signed_blob, timestamp_now(), )) } diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index fa171cd04bf..2c8e0f047fc 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -56,9 +56,7 @@ use std::ops::Sub; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; -use types::{ - BlobsSidecar, EthSpec, Hash256, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar, Slot, -}; +use types::{BlobSidecar, EthSpec, Hash256, SignedBeaconBlock, Slot}; /// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync /// from a peer. If a peer is within this tolerance (forwards or backwards), it is treated as a @@ -106,15 +104,7 @@ pub enum SyncMessage { RpcBlobs { request_id: RequestId, peer_id: PeerId, - blob_sidecar: Option>>, - seen_timestamp: Duration, - }, - - /// A block and blobs have been received from the RPC. - RpcBlockAndBlobs { - request_id: RequestId, - peer_id: PeerId, - block_and_blobs: Option>, + blob_sidecar: Option>>, seen_timestamp: Duration, }, @@ -654,17 +644,6 @@ impl SyncManager { blob_sidecar, seen_timestamp, } => self.rpc_blobs_received(request_id, peer_id, blob_sidecar, seen_timestamp), - SyncMessage::RpcBlockAndBlobs { - request_id, - peer_id, - block_and_blobs, - seen_timestamp, - } => self.rpc_block_block_and_blobs_received( - request_id, - peer_id, - block_and_blobs, - seen_timestamp, - ), } } @@ -897,7 +876,7 @@ impl SyncManager { &mut self, request_id: RequestId, peer_id: PeerId, - maybe_sidecar: Option::EthSpec>>>, + maybe_blob: Option::EthSpec>>>, _seen_timestamp: Duration, ) { match request_id { @@ -907,48 +886,17 @@ impl SyncManager { RequestId::BackFillBlocks { .. } => { unreachable!("An only blocks request does not receive sidecars") } - RequestId::BackFillBlobs { id } => { - self.blobs_backfill_response(id, peer_id, maybe_sidecar.into()) + RequestId::BackFillBlobs { .. } => { + unimplemented!("Adjust backfill sync"); } RequestId::RangeBlocks { .. } => { unreachable!("Only-blocks range requests don't receive sidecars") } RequestId::RangeBlobs { id } => { - self.blobs_range_response(id, peer_id, maybe_sidecar.into()) + unimplemented!("Adjust range"); } } } - - fn rpc_block_block_and_blobs_received( - &mut self, - request_id: RequestId, - peer_id: PeerId, - block_sidecar_pair: Option>, - seen_timestamp: Duration, - ) { - match request_id { - RequestId::SingleBlock { id } => self.block_lookups.single_block_lookup_response( - id, - peer_id, - block_sidecar_pair.map(|block_sidecar_pair| block_sidecar_pair.into()), - seen_timestamp, - &mut self.network, - ), - RequestId::ParentLookup { id } => self.block_lookups.parent_lookup_response( - id, - peer_id, - block_sidecar_pair.map(|block_sidecar_pair| block_sidecar_pair.into()), - seen_timestamp, - &mut self.network, - ), - RequestId::BackFillBlocks { .. } - | RequestId::BackFillBlobs { .. } - | RequestId::RangeBlocks { .. } - | RequestId::RangeBlobs { .. } => unreachable!( - "since range requests are not block-glob coupled, this should never be reachable" - ), - } - } } impl From>> for BlockProcessResult { diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index ef8f872cbfa..2da6a41e240 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -426,7 +426,7 @@ impl SyncNetworkContext { "count" => request.block_roots.len(), "peer" => %peer_id ); - Request::BlobsByRoot(request.into()) + unimplemented!("There is no longer such thing as a single block lookup, since we nede to ask for blobs and blocks separetely"); } else { trace!( self.log, @@ -467,7 +467,9 @@ impl SyncNetworkContext { "count" => request.block_roots.len(), "peer" => %peer_id ); - Request::BlobsByRoot(request.into()) + unimplemented!( + "Parent requests now need to interleave blocks and blobs or something like that." + ) } else { trace!( self.log, From ae3e5f73d6696a9f2ae41aafa9dd86502383068d Mon Sep 17 00:00:00 2001 From: Diva M Date: Fri, 10 Mar 2023 11:23:45 -0500 Subject: [PATCH 08/96] fmt --- .../beacon_processor/worker/rpc_methods.rs | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 3fb8feaf488..4480f37130e 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -246,7 +246,7 @@ impl Worker { for (known_index, blob) in blob_bundle.into_iter().enumerate() { if (known_index as u64) == index { let blob_sidecar = types::BlobSidecar{ - block_root: beacon_block_root, + block_root: beacon_block_root, index, slot: beacon_block_slot, block_parent_root: block.parent_root, @@ -839,18 +839,23 @@ impl Worker { match self.chain.get_blobs(&root, data_availability_boundary) { Ok(Some(blobs)) => { // TODO: more GROSS code ahead. Reader beware - let types::BlobsSidecar{ beacon_block_root, beacon_block_slot, blobs: blob_bundle, kzg_aggregated_proof }: types::BlobsSidecar<_> = blobs; + let types::BlobsSidecar { + beacon_block_root, + beacon_block_slot, + blobs: blob_bundle, + kzg_aggregated_proof, + }: types::BlobsSidecar<_> = blobs; for (blob_index, blob) in blob_bundle.into_iter().enumerate() { - let blob_sidecar = types::BlobSidecar{ - block_root: beacon_block_root, - index: blob_index as u64, - slot: beacon_block_slot, - block_parent_root: Hash256::zero(), - proposer_index: 0, - blob, - kzg_commitment: types::KzgCommitment::default(), - kzg_proof: types::KzgProof::default(), + let blob_sidecar = types::BlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot: beacon_block_slot, + block_parent_root: Hash256::zero(), + proposer_index: 0, + blob, + kzg_commitment: types::KzgCommitment::default(), + kzg_proof: types::KzgProof::default(), }; blobs_sent += 1; From 6ec0ce60702939762c5e17d67d4149fbe64f69df Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 15 Feb 2023 16:44:13 +1100 Subject: [PATCH 09/96] Implement get validator block endpoint for EIP-4844 --- beacon_node/beacon_chain/src/beacon_chain.rs | 47 ++++++++++------ beacon_node/beacon_chain/src/blob_cache.rs | 8 +-- beacon_node/beacon_chain/src/errors.rs | 1 + .../http_api/src/build_block_contents.rs | 34 +++++++++++ beacon_node/http_api/src/lib.rs | 6 +- beacon_node/http_api/src/publish_blocks.rs | 11 ++-- common/eth2/src/lib.rs | 4 +- consensus/types/src/beacon_block.rs | 11 ++++ .../src/beacon_block_and_blob_sidecars.rs | 37 ++++++++++++ consensus/types/src/blob_sidecar.rs | 6 ++ consensus/types/src/block_contents.rs | 56 +++++++++++++++++++ consensus/types/src/lib.rs | 6 +- validator_client/src/block_service.rs | 8 ++- 13 files changed, 201 insertions(+), 34 deletions(-) create mode 100644 beacon_node/http_api/src/build_block_contents.rs create mode 100644 consensus/types/src/beacon_block_and_blob_sidecars.rs create mode 100644 consensus/types/src/block_contents.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 1e901987a50..27719410560 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4759,30 +4759,41 @@ impl BeaconChain { .kzg .as_ref() .ok_or(BlockProductionError::TrustedSetupNotInitialized)?; - let kzg_aggregated_proof = - kzg_utils::compute_aggregate_kzg_proof::(kzg, &blobs) - .map_err(BlockProductionError::KzgError)?; let beacon_block_root = block.canonical_root(); let expected_kzg_commitments = block.body().blob_kzg_commitments().map_err(|_| { BlockProductionError::InvalidBlockVariant( "EIP4844 block does not contain kzg commitments".to_string(), ) })?; - let blobs_sidecar = BlobsSidecar { - beacon_block_slot: slot, - beacon_block_root, - blobs, - kzg_aggregated_proof, - }; - kzg_utils::validate_blobs_sidecar( - kzg, - slot, - beacon_block_root, - expected_kzg_commitments, - &blobs_sidecar, - ) - .map_err(BlockProductionError::KzgError)?; - self.blob_cache.put(beacon_block_root, blobs_sidecar); + + let blob_sidecars = VariableList::from( + blobs + .into_iter() + .enumerate() + .map(|(blob_index, blob)| { + BlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index, + blob, + kzg_commitment: expected_kzg_commitments[blob_index].clone(), + kzg_proof: Default::default(), // TODO: compute KZG proof + } + }) + .collect::>>(), + ); + + // TODO: validate blobs + // kzg_utils::validate_blobs_sidecar( + // kzg, + // slot, + // beacon_block_root, + // expected_kzg_commitments, + // &blobs_sidecar, + // ).map_err(BlockProductionError::KzgError)?; + self.blob_cache.put(beacon_block_root, blob_sidecars); } metrics::inc_counter(&metrics::BLOCK_PRODUCTION_SUCCESSES); diff --git a/beacon_node/beacon_chain/src/blob_cache.rs b/beacon_node/beacon_chain/src/blob_cache.rs index d03e62ab646..e3b0a06b64a 100644 --- a/beacon_node/beacon_chain/src/blob_cache.rs +++ b/beacon_node/beacon_chain/src/blob_cache.rs @@ -1,12 +1,12 @@ use lru::LruCache; use parking_lot::Mutex; -use types::{BlobsSidecar, EthSpec, Hash256}; +use types::{BlobSidecars, EthSpec, Hash256}; pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10; /// A cache blobs by beacon block root. pub struct BlobCache { - blobs: Mutex>>, + blobs: Mutex>>, } #[derive(Hash, PartialEq, Eq)] @@ -21,11 +21,11 @@ impl Default for BlobCache { } impl BlobCache { - pub fn put(&self, beacon_block: Hash256, blobs: BlobsSidecar) -> Option> { + pub fn put(&self, beacon_block: Hash256, blobs: BlobSidecars) -> Option> { self.blobs.lock().put(BlobCacheId(beacon_block), blobs) } - pub fn pop(&self, root: &Hash256) -> Option> { + pub fn pop(&self, root: &Hash256) -> Option> { self.blobs.lock().pop(&BlobCacheId(*root)) } } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 3b4c462494e..8f9aa02939d 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -266,6 +266,7 @@ pub enum BlockProductionError { blob_block_hash: ExecutionBlockHash, payload_block_hash: ExecutionBlockHash, }, + NoBlobsCached, FailedToReadFinalizedBlock(store::Error), MissingFinalizedBlock(Hash256), BlockTooLarge(usize), diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs new file mode 100644 index 00000000000..d456c320a63 --- /dev/null +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -0,0 +1,34 @@ +use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProductionError}; +use std::sync::Arc; +use types::{ + AbstractExecPayload, BeaconBlock, BeaconBlockAndBlobSidecars, BlockContents, ForkName, +}; + +type Error = warp::reject::Rejection; + +pub fn build_block_contents>( + fork_name: ForkName, + chain: Arc>, + block: BeaconBlock, +) -> Result, Error> { + match fork_name { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + Ok(BlockContents::Block(block)) + } + ForkName::Eip4844 => { + let block_root = &block.canonical_root(); + if let Some(blob_sidecars) = chain.blob_cache.pop(block_root) { + let block_and_blobs = BeaconBlockAndBlobSidecars { + block, + blob_sidecars, + }; + + Ok(BlockContents::BlockAndBlobSidecars(block_and_blobs)) + } else { + return Err(warp_utils::reject::block_production_error( + BlockProductionError::NoBlobsCached, + )); + } + } + } +} diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index a089b6e9747..ea398590663 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -11,6 +11,7 @@ mod attester_duties; mod block_id; mod block_packing_efficiency; mod block_rewards; +mod build_block_contents; mod database; mod metrics; mod proposer_duties; @@ -2421,7 +2422,10 @@ pub fn serve( .fork_name(&chain.spec) .map_err(inconsistent_fork_rejection)?; - fork_versioned_response(endpoint_version, fork_name, block) + let block_contents = + build_block_contents::build_block_contents(fork_name, chain, block); + + fork_versioned_response(endpoint_version, fork_name, block_contents?) .map(|response| warp::reply::json(&response)) }, ); diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 32494367363..1e785cd1ee7 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -12,7 +12,7 @@ use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ AbstractExecPayload, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload, - Hash256, SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar, + Hash256, SignedBeaconBlock, }; use warp::Rejection; @@ -40,10 +40,11 @@ pub async fn publish_block( let wrapped_block: BlockWrapper = if matches!(block.as_ref(), &SignedBeaconBlock::Eip4844(_)) { if let Some(sidecar) = chain.blob_cache.pop(&block_root) { - let block_and_blobs = SignedBeaconBlockAndBlobsSidecar { - beacon_block: block, - blobs_sidecar: Arc::new(sidecar), - }; + // TODO: Needs to be adjusted + // let block_and_blobs = SignedBeaconBlockAndBlobsSidecar { + // beacon_block: block, + // blobs_sidecar: Arc::new(sidecar), + // }; unimplemented!("Needs to be adjusted") } else { //FIXME(sean): This should probably return a specific no-blob-cached error code, beacon API coordination required diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 4535546a94b..5a55f21e3da 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -1388,7 +1388,7 @@ impl BeaconNodeHttpClient { slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blocks_modular(slot, randao_reveal, graffiti, SkipRandaoVerification::No) .await } @@ -1400,7 +1400,7 @@ impl BeaconNodeHttpClient { randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V2)?; path.path_segments_mut() diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 0f26cd0e5e7..a929265824a 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -731,6 +731,17 @@ impl From>> } } +impl> From> + for BeaconBlock +{ + fn from(block_contents: BlockContents) -> Self { + match block_contents { + BlockContents::BlockAndBlobSidecars(block_and_sidecars) => block_and_sidecars.block, + BlockContents::Block(block) => block, + } + } +} + impl> ForkVersionDeserialize for BeaconBlock { diff --git a/consensus/types/src/beacon_block_and_blob_sidecars.rs b/consensus/types/src/beacon_block_and_blob_sidecars.rs new file mode 100644 index 00000000000..c518f765b18 --- /dev/null +++ b/consensus/types/src/beacon_block_and_blob_sidecars.rs @@ -0,0 +1,37 @@ +use crate::{ + AbstractExecPayload, BeaconBlock, BlobSidecars, EthSpec, ForkName, ForkVersionDeserialize, +}; +use derivative::Derivative; +use serde_derive::{Deserialize, Serialize}; +use ssz_derive::Encode; +use tree_hash_derive::TreeHash; + +#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)] +#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] +#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] +pub struct BeaconBlockAndBlobSidecars> { + pub block: BeaconBlock, + pub blob_sidecars: BlobSidecars, +} + +impl> ForkVersionDeserialize + for BeaconBlockAndBlobSidecars +{ + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::value::Value, + fork_name: ForkName, + ) -> Result { + #[derive(Deserialize)] + #[serde(bound = "T: EthSpec")] + struct Helper { + block: serde_json::Value, + blob_sidecars: BlobSidecars, + } + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + + Ok(Self { + block: BeaconBlock::deserialize_by_fork::<'de, D>(helper.block, fork_name)?, + blob_sidecars: helper.blob_sidecars, + }) + } +} diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 12a6633dee6..251bef03436 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -5,6 +5,7 @@ use kzg::{KzgCommitment, KzgProof}; use serde_derive::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; +use ssz_types::VariableList; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; @@ -34,15 +35,20 @@ pub struct BlobIdentifier { pub struct BlobSidecar { pub block_root: Hash256, // TODO: fix the type, should fit in u8 as well + #[serde(with = "eth2_serde_utils::quoted_u64")] pub index: u64, pub slot: Slot, pub block_parent_root: Hash256, + #[serde(with = "eth2_serde_utils::quoted_u64")] pub proposer_index: u64, + #[serde(with = "ssz_types::serde_utils::hex_fixed_vec")] pub blob: Blob, pub kzg_commitment: KzgCommitment, pub kzg_proof: KzgProof, } +pub type BlobSidecars = VariableList, ::MaxBlobsPerBlock>; + impl SignedRoot for BlobSidecar {} impl BlobSidecar { diff --git a/consensus/types/src/block_contents.rs b/consensus/types/src/block_contents.rs new file mode 100644 index 00000000000..d5a500280c9 --- /dev/null +++ b/consensus/types/src/block_contents.rs @@ -0,0 +1,56 @@ +use crate::{ + AbstractExecPayload, BeaconBlock, BeaconBlockAndBlobSidecars, BlobSidecars, EthSpec, ForkName, + ForkVersionDeserialize, +}; +use derivative::Derivative; +use serde_derive::{Deserialize, Serialize}; + +/// A wrapper over a [`BeaconBlock`] or a [`BeaconBlockAndBlobSidecars`]. +#[derive(Clone, Debug, Derivative, Serialize, Deserialize)] +#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] +#[serde(untagged)] +#[serde(bound = "T: EthSpec")] +pub enum BlockContents> { + BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), + Block(BeaconBlock), +} + +impl> BlockContents { + pub fn block(&self) -> &BeaconBlock { + match self { + BlockContents::BlockAndBlobSidecars(block_and_sidecars) => &block_and_sidecars.block, + BlockContents::Block(block) => block, + } + } + + pub fn deconstruct(self) -> (BeaconBlock, Option>) { + match self { + BlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( + block_and_sidecars.block, + Some(block_and_sidecars.blob_sidecars), + ), + BlockContents::Block(block) => (block, None), + } + } +} + +impl> ForkVersionDeserialize + for BlockContents +{ + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::value::Value, + fork_name: ForkName, + ) -> Result { + match fork_name { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + Ok(BlockContents::Block(BeaconBlock::deserialize_by_fork::< + 'de, + D, + >(value, fork_name)?)) + } + ForkName::Eip4844 => Ok(BlockContents::BlockAndBlobSidecars( + BeaconBlockAndBlobSidecars::deserialize_by_fork::<'de, D>(value, fork_name)?, + )), + } + } +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index ce483785322..50b9547c9c4 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -99,8 +99,10 @@ pub mod slot_data; #[cfg(feature = "sqlite")] pub mod sqlite; +pub mod beacon_block_and_blob_sidecars; pub mod blob_sidecar; pub mod blobs_sidecar; +pub mod block_contents; pub mod signed_blob; pub mod signed_block_and_blobs; pub mod transaction; @@ -116,6 +118,7 @@ pub use crate::beacon_block::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockEip4844, BeaconBlockMerge, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, EmptyBlock, }; +pub use crate::beacon_block_and_blob_sidecars::BeaconBlockAndBlobSidecars; pub use crate::beacon_block_body::{ BeaconBlockBody, BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyCapella, BeaconBlockBodyEip4844, BeaconBlockBodyMerge, BeaconBlockBodyRef, BeaconBlockBodyRefMut, @@ -123,8 +126,9 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; -pub use crate::blob_sidecar::BlobSidecar; +pub use crate::blob_sidecar::{BlobSidecar, BlobSidecars}; pub use crate::blobs_sidecar::{Blobs, BlobsSidecar}; +pub use crate::block_contents::BlockContents; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 3b37492377f..22064cb1022 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -15,8 +15,8 @@ use std::time::Duration; use tokio::sync::mpsc; use tokio::time::sleep; use types::{ - AbstractExecPayload, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, PublicKeyBytes, - Slot, + AbstractExecPayload, BeaconBlock, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, + PublicKeyBytes, Slot, }; #[derive(Debug)] @@ -347,7 +347,7 @@ impl BlockService { RequireSynced::No, OfflineOnFailure::Yes, |beacon_node| async move { - let block = match Payload::block_type() { + let block: BeaconBlock = match Payload::block_type() { BlockType::Full => { let _get_timer = metrics::start_timer_vec( &metrics::BLOCK_SERVICE_TIMES, @@ -367,6 +367,7 @@ impl BlockService { )) })? .data + .into() } BlockType::Blinded => { let _get_timer = metrics::start_timer_vec( @@ -387,6 +388,7 @@ impl BlockService { )) })? .data + .into() } }; From e81a5029b8e6f6b4a2a72d381e62b7dc961ec8ac Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 14 Mar 2023 11:48:58 +1100 Subject: [PATCH 10/96] Update SignedBlobSidecar container --- beacon_node/lighthouse_network/src/types/pubsub.rs | 2 +- .../network/src/beacon_processor/worker/gossip_methods.rs | 4 ++-- consensus/types/src/signed_blob.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 243e8314850..87012859afa 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -320,7 +320,7 @@ impl std::fmt::Display for PubsubMessage { PubsubMessage::BlobSidecar(data) => write!( f, "BlobSidecar: slot: {}, blob index: {}", - data.1.blob.slot, data.1.blob.index, + data.1.message.slot, data.1.message.index, ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index 0bdebd88fef..15f9c5e29e1 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -664,8 +664,8 @@ impl Worker { "peer_id" => %peer_id, "client" => %peer_client, "blob_topic" => blob_index, - "blob_index" => signed_blob.blob.index, - "blob_slot" => signed_blob.blob.slot + "blob_index" => signed_blob.message.index, + "blob_slot" => signed_blob.message.slot ); } diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index a3cfb9d58af..8e2e47676b7 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -19,6 +19,6 @@ use tree_hash_derive::TreeHash; #[serde(bound = "T: EthSpec")] #[arbitrary(bound = "T: EthSpec")] pub struct SignedBlobSidecar { - pub blob: BlobSidecar, + pub message: BlobSidecar, pub signature: Signature, } From a8978a5f69e9752f283b1598b438acb6d6f7d89d Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 14 Mar 2023 17:03:45 +1100 Subject: [PATCH 11/96] Implement publish block and blobs endpoint (WIP) --- beacon_node/http_api/src/lib.rs | 6 +-- beacon_node/http_api/src/publish_blocks.rs | 27 ++++--------- common/eth2/src/lib.rs | 4 +- consensus/types/src/lib.rs | 5 ++- consensus/types/src/signed_blob.rs | 3 ++ consensus/types/src/signed_block_and_blobs.rs | 12 +++++- consensus/types/src/signed_block_contents.rs | 40 +++++++++++++++++++ validator_client/src/block_service.rs | 21 +++++----- 8 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 consensus/types/src/signed_block_contents.rs diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index ea398590663..aa4d5be718f 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -60,7 +60,7 @@ use types::{ ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, SignedBeaconBlock, SignedBlindedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, - SyncCommitteeMessage, SyncContributionData, + SyncCommitteeMessage, SyncContributionData, SignedBlockContents, }; use version::{ add_consensus_version_header, execution_optimistic_fork_versioned_response, @@ -1120,11 +1120,11 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |block: Arc>, + |block_contents: SignedBlockContents, chain: Arc>, network_tx: UnboundedSender>, log: Logger| async move { - publish_blocks::publish_block(None, block, chain, &network_tx, log) + publish_blocks::publish_block(None, block_contents, chain, &network_tx, log) .await .map(|()| warp::reply()) }, diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 1e785cd1ee7..d73ec437bbe 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -10,21 +10,20 @@ use slot_clock::SlotClock; use std::sync::Arc; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; -use types::{ - AbstractExecPayload, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload, - Hash256, SignedBeaconBlock, -}; +use types::{AbstractExecPayload, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload, Hash256, SignedBeaconBlock, SignedBlockContents}; use warp::Rejection; /// Handles a request from the HTTP API for full blocks. pub async fn publish_block( block_root: Option, - block: Arc>, + block_contents: SignedBlockContents, chain: Arc>, network_tx: &UnboundedSender>, log: Logger, ) -> Result<(), Rejection> { let seen_timestamp = timestamp_now(); + let (block, _maybe_blobs) = block_contents.deconstruct(); + let block = Arc::new(block); //FIXME(sean) have to move this to prior to publishing because it's included in the blobs sidecar message. //this may skew metrics @@ -38,20 +37,8 @@ pub async fn publish_block( // Send the block, regardless of whether or not it is valid. The API // specification is very clear that this is the desired behaviour. let wrapped_block: BlockWrapper = - if matches!(block.as_ref(), &SignedBeaconBlock::Eip4844(_)) { - if let Some(sidecar) = chain.blob_cache.pop(&block_root) { - // TODO: Needs to be adjusted - // let block_and_blobs = SignedBeaconBlockAndBlobsSidecar { - // beacon_block: block, - // blobs_sidecar: Arc::new(sidecar), - // }; - unimplemented!("Needs to be adjusted") - } else { - //FIXME(sean): This should probably return a specific no-blob-cached error code, beacon API coordination required - return Err(warp_utils::reject::broadcast_without_import( - "no blob cached for block".into(), - )); - } + if matches!(block.as_ref(), SignedBeaconBlock::Eip4844(_)) { + todo!("to be implemented") } else { crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; block.into() @@ -180,7 +167,7 @@ pub async fn publish_blinded_block( let full_block = reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; publish_block::( Some(block_root), - Arc::new(full_block), + SignedBlockContents::Block(full_block), chain, network_tx, log, diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 5a55f21e3da..2a27d31da9b 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -612,7 +612,7 @@ impl BeaconNodeHttpClient { /// Returns `Ok(None)` on a 404 error. pub async fn post_beacon_blocks>( &self, - block: &SignedBeaconBlock, + block_contents: &SignedBlockContents, ) -> Result<(), Error> { let mut path = self.eth_path(V1)?; @@ -621,7 +621,7 @@ impl BeaconNodeHttpClient { .push("beacon") .push("blocks"); - self.post_with_timeout(path, block, self.timeouts.proposal) + self.post_with_timeout(path, block_contents, self.timeouts.proposal) .await?; Ok(()) diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 50b9547c9c4..a8cd518f543 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -106,6 +106,7 @@ pub mod block_contents; pub mod signed_blob; pub mod signed_block_and_blobs; pub mod transaction; +pub mod signed_block_contents; use ethereum_types::{H160, H256}; @@ -187,8 +188,8 @@ pub use crate::signed_beacon_block::{ }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_blob::*; -pub use crate::signed_block_and_blobs::SignedBeaconBlockAndBlobsSidecar; -pub use crate::signed_block_and_blobs::SignedBeaconBlockAndBlobsSidecarDecode; +pub use crate::signed_block_and_blobs::{SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockAndBlobsSidecarDecode, SignedBeaconBlockAndBlobSidecars}; +pub use crate::signed_block_contents::SignedBlockContents; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; pub use crate::signed_contribution_and_proof::SignedContributionAndProof; pub use crate::signed_voluntary_exit::SignedVoluntaryExit; diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index 8e2e47676b7..d7d2f884d49 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -3,6 +3,7 @@ use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; +use derivative::Derivative; #[derive( Debug, @@ -14,10 +15,12 @@ use tree_hash_derive::TreeHash; Decode, TestRandom, TreeHash, + Derivative, arbitrary::Arbitrary, )] #[serde(bound = "T: EthSpec")] #[arbitrary(bound = "T: EthSpec")] +#[derivative(Hash(bound = "T: EthSpec"))] pub struct SignedBlobSidecar { pub message: BlobSidecar, pub signature: Signature, diff --git a/consensus/types/src/signed_block_and_blobs.rs b/consensus/types/src/signed_block_and_blobs.rs index 4ea0d6616db..3ab3d3dfee2 100644 --- a/consensus/types/src/signed_block_and_blobs.rs +++ b/consensus/types/src/signed_block_and_blobs.rs @@ -1,9 +1,10 @@ -use crate::{BlobsSidecar, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844}; +use crate::{AbstractExecPayload, BlobsSidecar, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844, SignedBlobSidecar}; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz::{Decode, DecodeError}; use ssz_derive::{Decode, Encode}; use std::sync::Arc; +use ssz_types::VariableList; use tree_hash_derive::TreeHash; #[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, PartialEq)] @@ -13,6 +14,7 @@ pub struct SignedBeaconBlockAndBlobsSidecarDecode { pub blobs_sidecar: BlobsSidecar, } +// TODO: will be removed once we decouple blobs in Gossip #[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)] #[derivative(PartialEq, Hash(bound = "T: EthSpec"))] pub struct SignedBeaconBlockAndBlobsSidecar { @@ -32,3 +34,11 @@ impl SignedBeaconBlockAndBlobsSidecar { }) } } + +#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)] +#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] +#[serde(bound = "T: EthSpec")] +pub struct SignedBeaconBlockAndBlobSidecars> { + pub signed_block: SignedBeaconBlock, + pub signed_blob_sidecars: VariableList, ::MaxBlobsPerBlock>, +} \ No newline at end of file diff --git a/consensus/types/src/signed_block_contents.rs b/consensus/types/src/signed_block_contents.rs new file mode 100644 index 00000000000..a6743c60f17 --- /dev/null +++ b/consensus/types/src/signed_block_contents.rs @@ -0,0 +1,40 @@ +use crate::{AbstractExecPayload, EthSpec, FullPayload, SignedBeaconBlock, SignedBlobSidecar}; +use crate::signed_block_and_blobs::SignedBeaconBlockAndBlobSidecars; +use derivative::Derivative; +use serde_derive::{Deserialize, Serialize}; +use ssz_types::VariableList; + +/// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. +#[derive(Clone, Debug, Derivative, Serialize, Deserialize)] +#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] +#[serde(untagged)] +#[serde(bound = "T: EthSpec")] +pub enum SignedBlockContents = FullPayload> { + BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars), + Block(SignedBeaconBlock), +} + +impl> SignedBlockContents { + pub fn signed_block(&self) -> &SignedBeaconBlock { + match self { + SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => &block_and_sidecars.signed_block, + SignedBlockContents::Block(block) => block, + } + } + + pub fn deconstruct(self) -> (SignedBeaconBlock, Option, ::MaxBlobsPerBlock>>) { + match self { + SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( + block_and_sidecars.signed_block, + Some(block_and_sidecars.signed_blob_sidecars), + ), + SignedBlockContents::Block(block) => (block, None), + } + } +} + +impl> From> for SignedBlockContents { + fn from(block: SignedBeaconBlock) -> Self { + SignedBlockContents::Block(block) + } +} \ No newline at end of file diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 22064cb1022..0cc0e40a1a8 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -14,10 +14,7 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use tokio::time::sleep; -use types::{ - AbstractExecPayload, BeaconBlock, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, - PublicKeyBytes, Slot, -}; +use types::{AbstractExecPayload, BeaconBlock, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, PublicKeyBytes, SignedBlockContents, Slot}; #[derive(Debug)] pub enum BlockError { @@ -410,11 +407,12 @@ impl BlockService { .await?; let signing_timer = metrics::start_timer(&metrics::BLOCK_SIGNING_TIMES); - let signed_block = self_ref + let signed_block_contents: SignedBlockContents = self_ref .validator_store .sign_block::(*validator_pubkey_ref, block, current_slot) .await - .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))?; + .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))? + .into(); let signing_time_ms = Duration::from_secs_f64(signing_timer.map_or(0.0, |t| t.stop_and_record())).as_millis(); @@ -438,7 +436,7 @@ impl BlockService { &[metrics::BEACON_BLOCK_HTTP_POST], ); beacon_node - .post_beacon_blocks(&signed_block) + .post_beacon_blocks(&signed_block_contents) .await .map_err(|e| { BlockError::Irrecoverable(format!( @@ -453,7 +451,8 @@ impl BlockService { &[metrics::BLINDED_BEACON_BLOCK_HTTP_POST], ); beacon_node - .post_beacon_blinded_blocks(&signed_block) + // TODO: need to be adjusted for blobs + .post_beacon_blinded_blocks(&signed_block_contents.signed_block()) .await .map_err(|e| { BlockError::Irrecoverable(format!( @@ -472,10 +471,10 @@ impl BlockService { log, "Successfully published block"; "block_type" => ?Payload::block_type(), - "deposits" => signed_block.message().body().deposits().len(), - "attestations" => signed_block.message().body().attestations().len(), + "deposits" => signed_block_contents.signed_block().message().body().deposits().len(), + "attestations" => signed_block_contents.signed_block().message().body().attestations().len(), "graffiti" => ?graffiti.map(|g| g.as_utf8_lossy()), - "slot" => signed_block.slot().as_u64(), + "slot" => signed_block_contents.signed_block().slot().as_u64(), ); Ok(()) From 76f49bdb44f1769e56234d430da48e662b413fb4 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 14 Mar 2023 12:13:15 +0530 Subject: [PATCH 12/96] Update kzg interface (#4077) * Update kzg interface * Update utils * Update dependency * Address review comments --- Cargo.lock | 3 +- beacon_node/beacon_chain/src/kzg_utils.rs | 65 +++++++++--------- crypto/kzg/Cargo.toml | 2 +- crypto/kzg/src/kzg_commitment.rs | 25 ++++--- crypto/kzg/src/kzg_proof.rs | 35 +++++----- crypto/kzg/src/lib.rs | 82 +++++++++++++++-------- 6 files changed, 127 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db377122218..0e669154f5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -892,10 +892,11 @@ dependencies = [ [[package]] name = "c-kzg" version = "0.1.0" -source = "git+https://github.com/ethereum/c-kzg-4844?rev=69f6155d7524247be9d3f54ab3bfbe33a0345622#69f6155d7524247be9d3f54ab3bfbe33a0345622" +source = "git+https://github.com/ethereum/c-kzg-4844?rev=549739fcb3aaec6fe5651e1912f05c604b45621b#549739fcb3aaec6fe5651e1912f05c604b45621b" dependencies = [ "hex", "libc", + "serde", ] [[package]] diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 58ff79e0f33..3536420a161 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -1,6 +1,8 @@ use kzg::{Error as KzgError, Kzg, BYTES_PER_BLOB}; -use types::{Blob, BlobsSidecar, EthSpec, Hash256, KzgCommitment, KzgProof, Slot}; +use types::{Blob, EthSpec, KzgCommitment, KzgProof}; +/// Converts a blob ssz List object to an array to be used with the kzg +/// crypto library. fn ssz_blob_to_crypto_blob(blob: Blob) -> kzg::Blob { let blob_vec: Vec = blob.into(); let mut arr = [0; BYTES_PER_BLOB]; @@ -8,47 +10,48 @@ fn ssz_blob_to_crypto_blob(blob: Blob) -> kzg::Blob { arr.into() } -pub fn validate_blobs_sidecar( +/// Validate a single blob-commitment-proof triplet from a `BlobSidecar`. +pub fn validate_blob( kzg: &Kzg, - slot: Slot, - beacon_block_root: Hash256, - expected_kzg_commitments: &[KzgCommitment], - blobs_sidecar: &BlobsSidecar, + blob: Blob, + kzg_commitment: KzgCommitment, + kzg_proof: KzgProof, ) -> Result { - if slot != blobs_sidecar.beacon_block_slot - || beacon_block_root != blobs_sidecar.beacon_block_root - || blobs_sidecar.blobs.len() != expected_kzg_commitments.len() - { - return Ok(false); - } - - let blobs = blobs_sidecar - .blobs - .clone() // TODO(pawan): avoid this clone - .into_iter() - .map(|blob| ssz_blob_to_crypto_blob::(blob)) - .collect::>(); - - kzg.verify_aggregate_kzg_proof( - &blobs, - expected_kzg_commitments, - blobs_sidecar.kzg_aggregated_proof, + kzg.verify_blob_kzg_proof( + ssz_blob_to_crypto_blob::(blob), + kzg_commitment, + kzg_proof, ) } -pub fn compute_aggregate_kzg_proof( +/// Validate a batch of blob-commitment-proof triplets from multiple `BlobSidecars`. +pub fn validate_blobs( kzg: &Kzg, + expected_kzg_commitments: &[KzgCommitment], blobs: &[Blob], -) -> Result { + kzg_proofs: &[KzgProof], +) -> Result { let blobs = blobs .iter() - .map(|blob| ssz_blob_to_crypto_blob::(blob.clone())) // TODO(pawan): avoid this clone + .map(|blob| ssz_blob_to_crypto_blob::(blob.clone())) // Avoid this clone .collect::>(); - kzg.compute_aggregate_kzg_proof(&blobs) + kzg.verify_blob_kzg_proof_batch(&blobs, expected_kzg_commitments, kzg_proofs) +} + +/// Compute the kzg proof given an ssz blob and its kzg commitment. +pub fn compute_blob_kzg_proof( + kzg: &Kzg, + blob: Blob, + kzg_commitment: KzgCommitment, +) -> Result { + kzg.compute_blob_kzg_proof(ssz_blob_to_crypto_blob::(blob), kzg_commitment) } -pub fn blob_to_kzg_commitment(kzg: &Kzg, blob: Blob) -> KzgCommitment { - let blob = ssz_blob_to_crypto_blob::(blob); - kzg.blob_to_kzg_commitment(blob) +/// Compute the kzg commitment for a given blob. +pub fn blob_to_kzg_commitment( + kzg: &Kzg, + blob: Blob, +) -> Result { + kzg.blob_to_kzg_commitment(ssz_blob_to_crypto_blob::(blob)) } diff --git a/crypto/kzg/Cargo.toml b/crypto/kzg/Cargo.toml index d37fda1eaa5..0645b3a2b93 100644 --- a/crypto/kzg/Cargo.toml +++ b/crypto/kzg/Cargo.toml @@ -16,7 +16,7 @@ serde_derive = "1.0.116" eth2_serde_utils = "0.1.1" hex = "0.4.2" eth2_hashing = "0.3.0" -c-kzg = {git = "https://github.com/ethereum/c-kzg-4844", rev = "69f6155d7524247be9d3f54ab3bfbe33a0345622" } +c-kzg = {git = "https://github.com/ethereum/c-kzg-4844", rev = "549739fcb3aaec6fe5651e1912f05c604b45621b" } arbitrary = { version = "1.0", features = ["derive"], optional = true } [features] diff --git a/crypto/kzg/src/kzg_commitment.rs b/crypto/kzg/src/kzg_commitment.rs index 7a43d7009d0..3a18fe8b548 100644 --- a/crypto/kzg/src/kzg_commitment.rs +++ b/crypto/kzg/src/kzg_commitment.rs @@ -1,3 +1,4 @@ +use c_kzg::{Bytes48, BYTES_PER_COMMITMENT}; use derivative::Derivative; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; @@ -7,12 +8,16 @@ use std::fmt::{Debug, Display, Formatter}; use std::str::FromStr; use tree_hash::{PackedEncoding, TreeHash}; -const KZG_COMMITMENT_BYTES_LEN: usize = 48; - #[derive(Derivative, Clone, Encode, Decode)] #[derivative(PartialEq, Eq, Hash)] #[ssz(struct_behaviour = "transparent")] -pub struct KzgCommitment(pub [u8; KZG_COMMITMENT_BYTES_LEN]); +pub struct KzgCommitment(pub [u8; BYTES_PER_COMMITMENT]); + +impl From for Bytes48 { + fn from(value: KzgCommitment) -> Self { + value.0.into() + } +} impl Display for KzgCommitment { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { @@ -22,13 +27,13 @@ impl Display for KzgCommitment { impl Default for KzgCommitment { fn default() -> Self { - KzgCommitment([0; KZG_COMMITMENT_BYTES_LEN]) + KzgCommitment([0; BYTES_PER_COMMITMENT]) } } impl TreeHash for KzgCommitment { fn tree_hash_type() -> tree_hash::TreeHashType { - <[u8; KZG_COMMITMENT_BYTES_LEN] as TreeHash>::tree_hash_type() + <[u8; BYTES_PER_COMMITMENT] as TreeHash>::tree_hash_type() } fn tree_hash_packed_encoding(&self) -> PackedEncoding { @@ -36,7 +41,7 @@ impl TreeHash for KzgCommitment { } fn tree_hash_packing_factor() -> usize { - <[u8; KZG_COMMITMENT_BYTES_LEN] as TreeHash>::tree_hash_packing_factor() + <[u8; BYTES_PER_COMMITMENT] as TreeHash>::tree_hash_packing_factor() } fn tree_hash_root(&self) -> tree_hash::Hash256 { @@ -86,15 +91,15 @@ impl FromStr for KzgCommitment { fn from_str(s: &str) -> Result { if let Some(stripped) = s.strip_prefix("0x") { let bytes = hex::decode(stripped).map_err(|e| e.to_string())?; - if bytes.len() == KZG_COMMITMENT_BYTES_LEN { - let mut kzg_commitment_bytes = [0; KZG_COMMITMENT_BYTES_LEN]; + if bytes.len() == BYTES_PER_COMMITMENT { + let mut kzg_commitment_bytes = [0; BYTES_PER_COMMITMENT]; kzg_commitment_bytes[..].copy_from_slice(&bytes); Ok(Self(kzg_commitment_bytes)) } else { Err(format!( "InvalidByteLength: got {}, expected {}", bytes.len(), - KZG_COMMITMENT_BYTES_LEN + BYTES_PER_COMMITMENT )) } } else { @@ -112,7 +117,7 @@ impl Debug for KzgCommitment { #[cfg(feature = "arbitrary")] impl arbitrary::Arbitrary<'_> for KzgCommitment { fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { - let mut bytes = [0u8; KZG_COMMITMENT_BYTES_LEN]; + let mut bytes = [0u8; BYTES_PER_COMMITMENT]; u.fill_buffer(&mut bytes)?; Ok(KzgCommitment(bytes)) } diff --git a/crypto/kzg/src/kzg_proof.rs b/crypto/kzg/src/kzg_proof.rs index 2dac5669b7e..7c6eb59abf8 100644 --- a/crypto/kzg/src/kzg_proof.rs +++ b/crypto/kzg/src/kzg_proof.rs @@ -1,3 +1,4 @@ +use c_kzg::{Bytes48, BYTES_PER_PROOF}; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use ssz_derive::{Decode, Encode}; @@ -6,15 +7,19 @@ use std::fmt::Debug; use std::str::FromStr; use tree_hash::{PackedEncoding, TreeHash}; -const KZG_PROOF_BYTES_LEN: usize = 48; - #[derive(PartialEq, Hash, Clone, Copy, Encode, Decode)] #[ssz(struct_behaviour = "transparent")] -pub struct KzgProof(pub [u8; KZG_PROOF_BYTES_LEN]); +pub struct KzgProof(pub [u8; BYTES_PER_PROOF]); + +impl From for Bytes48 { + fn from(value: KzgProof) -> Self { + value.0.into() + } +} impl KzgProof { pub fn empty() -> Self { - let mut bytes = [0; KZG_PROOF_BYTES_LEN]; + let mut bytes = [0; BYTES_PER_PROOF]; bytes[0] = 192; Self(bytes) } @@ -28,25 +33,25 @@ impl fmt::Display for KzgProof { impl Default for KzgProof { fn default() -> Self { - KzgProof([0; KZG_PROOF_BYTES_LEN]) + KzgProof([0; BYTES_PER_PROOF]) } } -impl From<[u8; KZG_PROOF_BYTES_LEN]> for KzgProof { - fn from(bytes: [u8; KZG_PROOF_BYTES_LEN]) -> Self { +impl From<[u8; BYTES_PER_PROOF]> for KzgProof { + fn from(bytes: [u8; BYTES_PER_PROOF]) -> Self { Self(bytes) } } -impl Into<[u8; KZG_PROOF_BYTES_LEN]> for KzgProof { - fn into(self) -> [u8; KZG_PROOF_BYTES_LEN] { +impl Into<[u8; BYTES_PER_PROOF]> for KzgProof { + fn into(self) -> [u8; BYTES_PER_PROOF] { self.0 } } impl TreeHash for KzgProof { fn tree_hash_type() -> tree_hash::TreeHashType { - <[u8; KZG_PROOF_BYTES_LEN]>::tree_hash_type() + <[u8; BYTES_PER_PROOF]>::tree_hash_type() } fn tree_hash_packed_encoding(&self) -> PackedEncoding { @@ -54,7 +59,7 @@ impl TreeHash for KzgProof { } fn tree_hash_packing_factor() -> usize { - <[u8; KZG_PROOF_BYTES_LEN]>::tree_hash_packing_factor() + <[u8; BYTES_PER_PROOF]>::tree_hash_packing_factor() } fn tree_hash_root(&self) -> tree_hash::Hash256 { @@ -104,15 +109,15 @@ impl FromStr for KzgProof { fn from_str(s: &str) -> Result { if let Some(stripped) = s.strip_prefix("0x") { let bytes = hex::decode(stripped).map_err(|e| e.to_string())?; - if bytes.len() == KZG_PROOF_BYTES_LEN { - let mut kzg_proof_bytes = [0; KZG_PROOF_BYTES_LEN]; + if bytes.len() == BYTES_PER_PROOF { + let mut kzg_proof_bytes = [0; BYTES_PER_PROOF]; kzg_proof_bytes[..].copy_from_slice(&bytes); Ok(Self(kzg_proof_bytes)) } else { Err(format!( "InvalidByteLength: got {}, expected {}", bytes.len(), - KZG_PROOF_BYTES_LEN + BYTES_PER_PROOF )) } } else { @@ -130,7 +135,7 @@ impl Debug for KzgProof { #[cfg(feature = "arbitrary")] impl arbitrary::Arbitrary<'_> for KzgProof { fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { - let mut bytes = [0u8; KZG_PROOF_BYTES_LEN]; + let mut bytes = [0u8; BYTES_PER_PROOF]; u.fill_buffer(&mut bytes)?; Ok(KzgProof(bytes)) } diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index b6d62e053fb..cd28c8529a9 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -3,6 +3,7 @@ mod kzg_proof; mod trusted_setup; pub use crate::{kzg_commitment::KzgCommitment, kzg_proof::KzgProof, trusted_setup::TrustedSetup}; +use c_kzg::Bytes48; pub use c_kzg::{ Blob, Error as CKzgError, KZGSettings, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT, FIELD_ELEMENTS_PER_BLOB, @@ -13,9 +14,9 @@ use std::path::PathBuf; pub enum Error { InvalidTrustedSetup(CKzgError), InvalidKzgProof(CKzgError), - InvalidLength(String), + InvalidBytes(CKzgError), KzgProofComputationFailed(CKzgError), - InvalidBlob(String), + InvalidBlob(CKzgError), } /// A wrapper over a kzg library that holds the trusted setup parameters. @@ -51,40 +52,67 @@ impl Kzg { }) } - /// Compute the aggregated kzg proof given an array of blobs. - pub fn compute_aggregate_kzg_proof(&self, blobs: &[Blob]) -> Result { - c_kzg::KZGProof::compute_aggregate_kzg_proof(blobs, &self.trusted_setup) + /// Compute the kzg proof given a blob and its kzg commitment. + pub fn compute_blob_kzg_proof( + &self, + blob: Blob, + kzg_commitment: KzgCommitment, + ) -> Result { + c_kzg::KZGProof::compute_blob_kzg_proof(blob, kzg_commitment.into(), &self.trusted_setup) .map_err(Error::KzgProofComputationFailed) - .map(|proof| KzgProof(proof.to_bytes())) + .map(|proof| KzgProof(proof.to_bytes().into_inner())) } - /// Verify an aggregate kzg proof given the blobs that generated the proof, the kzg commitments - /// and the kzg proof. - pub fn verify_aggregate_kzg_proof( + /// Verify a kzg proof given the blob, kzg commitment and kzg proof. + pub fn verify_blob_kzg_proof( + &self, + blob: Blob, + kzg_commitment: KzgCommitment, + kzg_proof: KzgProof, + ) -> Result { + c_kzg::KZGProof::verify_blob_kzg_proof( + blob, + kzg_commitment.into(), + kzg_proof.into(), + &self.trusted_setup, + ) + .map_err(Error::InvalidKzgProof) + } + + /// Verify a batch of blob commitment proof triplets. + /// + /// Note: This method is slightly faster than calling `Self::verify_blob_kzg_proof` in a loop sequentially. + /// TODO(pawan): test performance against a parallelized rayon impl. + pub fn verify_blob_kzg_proof_batch( &self, blobs: &[Blob], - expected_kzg_commitments: &[KzgCommitment], - kzg_aggregated_proof: KzgProof, + kzg_commitments: &[KzgCommitment], + kzg_proofs: &[KzgProof], ) -> Result { - if blobs.len() != expected_kzg_commitments.len() { - return Err(Error::InvalidLength( - "blobs and expected_kzg_commitments should be of same size".to_string(), - )); - } - let commitments = expected_kzg_commitments + let commitments_bytes = kzg_commitments + .iter() + .map(|comm| Bytes48::from_bytes(&comm.0)) + .collect::, _>>() + .map_err(Error::InvalidBytes)?; + + let proofs_bytes = kzg_proofs .iter() - .map(|comm| comm.0.into()) - .collect::>(); - let proof: c_kzg::KZGProof = kzg_aggregated_proof.0.into(); - proof - .verify_aggregate_kzg_proof(blobs, &commitments, &self.trusted_setup) - .map_err(Error::InvalidKzgProof) + .map(|proof| Bytes48::from_bytes(&proof.0)) + .collect::, _>>() + .map_err(Error::InvalidBytes)?; + c_kzg::KZGProof::verify_blob_kzg_proof_batch( + blobs, + &commitments_bytes, + &proofs_bytes, + &self.trusted_setup, + ) + .map_err(Error::InvalidKzgProof) } /// Converts a blob to a kzg commitment. - pub fn blob_to_kzg_commitment(&self, blob: Blob) -> KzgCommitment { - KzgCommitment( - c_kzg::KZGCommitment::blob_to_kzg_commitment(blob, &self.trusted_setup).to_bytes(), - ) + pub fn blob_to_kzg_commitment(&self, blob: Blob) -> Result { + c_kzg::KZGCommitment::blob_to_kzg_commitment(blob, &self.trusted_setup) + .map_err(Error::InvalidBlob) + .map(|com| KzgCommitment(com.to_bytes().into_inner())) } } From 62627d984c155cd799b32f39d9d69a91b5f238e3 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 15 Mar 2023 12:30:59 +1100 Subject: [PATCH 13/96] Comment out code that fails to compile --- .../beacon_chain/src/blob_verification.rs | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 1d7f9cc3c52..34aaf643568 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -135,17 +135,20 @@ fn verify_data_availability( .as_ref() .ok_or(BlobError::TrustedSetupNotInitialized)?; - if !kzg_utils::validate_blobs_sidecar( - kzg, - block_slot, - block_root, - kzg_commitments, - blob_sidecar, - ) - .map_err(BlobError::KzgError)? - { - return Err(BlobError::InvalidKzgProof); - } + // TODO: use `kzg_utils::validate_blobs` once the function is updated + // I believe this is currently being worked on in another branch. + // + // if !kzg_utils::validate_blobs_sidecar( + // kzg, + // block_slot, + // block_root, + // kzg_commitments, + // blob_sidecar, + // ) + // .map_err(BlobError::KzgError)? + // { + // return Err(BlobError::InvalidKzgProof); + // } Ok(()) } From 02a88f07049c7f6d5de91542050f0b1d0006af69 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 15 Mar 2023 15:15:46 +1100 Subject: [PATCH 14/96] Add KZG proof and blob validation --- beacon_node/beacon_chain/src/beacon_chain.rs | 82 +++++++++++++++---- beacon_node/beacon_chain/src/errors.rs | 1 + beacon_node/beacon_chain/src/kzg_utils.rs | 5 +- .../lighthouse_network/src/types/topics.rs | 1 - beacon_node/store/src/hot_cold_store.rs | 4 +- crypto/kzg/src/kzg_commitment.rs | 2 +- 6 files changed, 71 insertions(+), 24 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 27719410560..57c68532573 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -73,6 +73,7 @@ use fork_choice::{ use futures::channel::mpsc::Sender; use itertools::process_results; use itertools::Itertools; +use kzg::Kzg; use operation_pool::{AttestationRef, OperationPool, PersistedOperationPool, ReceivedPreCapella}; use parking_lot::{Mutex, RwLock}; use proto_array::{CountUnrealizedFull, DoNotReOrg, ProposerHeadError}; @@ -107,6 +108,7 @@ use store::{ use task_executor::{ShutdownReason, TaskExecutor}; use tree_hash::TreeHash; use types::beacon_state::CloneConfig; +use types::blobs_sidecar::KzgCommitments; use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; use types::consts::merge::INTERVALS_PER_SLOT; use types::*; @@ -4760,39 +4762,60 @@ impl BeaconChain { .as_ref() .ok_or(BlockProductionError::TrustedSetupNotInitialized)?; let beacon_block_root = block.canonical_root(); - let expected_kzg_commitments = block.body().blob_kzg_commitments().map_err(|_| { - BlockProductionError::InvalidBlockVariant( - "EIP4844 block does not contain kzg commitments".to_string(), - ) - })?; + let expected_kzg_commitments: &KzgCommitments = + block.body().blob_kzg_commitments().map_err(|_| { + BlockProductionError::InvalidBlockVariant( + "EIP4844 block does not contain kzg commitments".to_string(), + ) + })?; + + if expected_kzg_commitments.len() != blobs.len() { + return Err(BlockProductionError::MissingKzgCommitment(format!( + "Missing KZG commitment for slot {}. Expected {}, got: {}", + slot, + blobs.len(), + expected_kzg_commitments.len() + ))); + } + + let kzg_proofs = + Self::compute_blob_kzg_proofs(kzg, &blobs, expected_kzg_commitments, slot)?; + + kzg_utils::validate_blobs::( + kzg, + expected_kzg_commitments, + &blobs, + &kzg_proofs, + ) + .map_err(BlockProductionError::KzgError)?; let blob_sidecars = VariableList::from( blobs .into_iter() .enumerate() .map(|(blob_index, blob)| { - BlobSidecar { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .expect("KZG commitment should exist for blob"); + + let kzg_proof = kzg_proofs + .get(blob_index) + .expect("KZG proof should exist for blob"); + + Ok(BlobSidecar { block_root: beacon_block_root, index: blob_index as u64, slot, block_parent_root: block.parent_root(), proposer_index, blob, - kzg_commitment: expected_kzg_commitments[blob_index].clone(), - kzg_proof: Default::default(), // TODO: compute KZG proof - } + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + }) }) - .collect::>>(), + .collect::>, BlockProductionError>>()?, ); - // TODO: validate blobs - // kzg_utils::validate_blobs_sidecar( - // kzg, - // slot, - // beacon_block_root, - // expected_kzg_commitments, - // &blobs_sidecar, - // ).map_err(BlockProductionError::KzgError)?; self.blob_cache.put(beacon_block_root, blob_sidecars); } @@ -4809,6 +4832,29 @@ impl BeaconChain { Ok((block, state)) } + fn compute_blob_kzg_proofs( + kzg: &Arc, + blobs: &Blobs<::EthSpec>, + expected_kzg_commitments: &KzgCommitments<::EthSpec>, + slot: Slot, + ) -> Result, BlockProductionError> { + blobs + .iter() + .enumerate() + .map(|(blob_index, blob)| { + let kzg_commitment = expected_kzg_commitments.get(blob_index).ok_or( + BlockProductionError::MissingKzgCommitment(format!( + "Missing KZG commitment for slot {} blob index {}", + slot, blob_index + )), + )?; + + kzg_utils::compute_blob_kzg_proof::(kzg, blob, kzg_commitment.clone()) + .map_err(BlockProductionError::KzgError) + }) + .collect::, BlockProductionError>>() + } + /// This method must be called whenever an execution engine indicates that a payload is /// invalid. /// diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 8f9aa02939d..911fe1d5b47 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -273,6 +273,7 @@ pub enum BlockProductionError { ShuttingDown, MissingSyncAggregate, MissingExecutionPayload, + MissingKzgCommitment(String), TokioJoin(tokio::task::JoinError), BeaconChain(BeaconChainError), InvalidPayloadFork, diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 3536420a161..19788a1b459 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -42,10 +42,11 @@ pub fn validate_blobs( /// Compute the kzg proof given an ssz blob and its kzg commitment. pub fn compute_blob_kzg_proof( kzg: &Kzg, - blob: Blob, + blob: &Blob, kzg_commitment: KzgCommitment, ) -> Result { - kzg.compute_blob_kzg_proof(ssz_blob_to_crypto_blob::(blob), kzg_commitment) + // Avoid this blob clone + kzg.compute_blob_kzg_proof(ssz_blob_to_crypto_blob::(blob.clone()), kzg_commitment) } /// Compute the kzg commitment for a given blob. diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 88764fb9514..d9deaaf0510 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -314,7 +314,6 @@ mod tests { VoluntaryExit, ProposerSlashing, AttesterSlashing, - BeaconBlocksAndBlobsSidecar, ] .iter() { diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 9762a0f59e8..60e2f775959 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -259,7 +259,7 @@ impl HotColdDB, LevelDB> { db.blobs_db = Some(LevelDB::open(path.as_path())?); } } - let blob_info = blob_info.unwrap_or(db.get_blob_info()); + let blob_info = blob_info.unwrap_or_else(|| db.get_blob_info()); db.compare_and_set_blob_info_with_write(blob_info, new_blob_info)?; info!( db.log, @@ -1899,7 +1899,7 @@ impl, Cold: ItemStore> HotColdDB let blob_info = self.get_blob_info(); let oldest_blob_slot = blob_info .oldest_blob_slot - .unwrap_or(eip4844_fork.start_slot(E::slots_per_epoch())); + .unwrap_or_else(|| eip4844_fork.start_slot(E::slots_per_epoch())); // The last entirely pruned epoch, blobs sidecar pruning may have stopped early in the // middle of an epoch otherwise the oldest blob slot is a start slot. diff --git a/crypto/kzg/src/kzg_commitment.rs b/crypto/kzg/src/kzg_commitment.rs index 3a18fe8b548..0f2725b7522 100644 --- a/crypto/kzg/src/kzg_commitment.rs +++ b/crypto/kzg/src/kzg_commitment.rs @@ -8,7 +8,7 @@ use std::fmt::{Debug, Display, Formatter}; use std::str::FromStr; use tree_hash::{PackedEncoding, TreeHash}; -#[derive(Derivative, Clone, Encode, Decode)] +#[derive(Derivative, Clone, Copy, Encode, Decode)] #[derivative(PartialEq, Eq, Hash)] #[ssz(struct_behaviour = "transparent")] pub struct KzgCommitment(pub [u8; BYTES_PER_COMMITMENT]); From 5887c8fe92b40afc8f2e4b5147fef2f2cc2538e7 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 15 Mar 2023 15:29:16 +1100 Subject: [PATCH 15/96] Update commented code to use todo! --- beacon_node/beacon_chain/src/blob_verification.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 34aaf643568..7a4493c9736 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -135,9 +135,7 @@ fn verify_data_availability( .as_ref() .ok_or(BlobError::TrustedSetupNotInitialized)?; - // TODO: use `kzg_utils::validate_blobs` once the function is updated - // I believe this is currently being worked on in another branch. - // + todo!("use `kzg_utils::validate_blobs` once the function is updated") // if !kzg_utils::validate_blobs_sidecar( // kzg, // block_slot, @@ -149,7 +147,7 @@ fn verify_data_availability( // { // return Err(BlobError::InvalidKzgProof); // } - Ok(()) + // Ok(()) } /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobsSidecar`]. This makes no From 775ca89801d7fb05b94a685d77e9e2956aaffb6a Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 15 Mar 2023 15:57:30 +1100 Subject: [PATCH 16/96] Add blobs publishing --- beacon_node/http_api/src/publish_blocks.rs | 31 ++++++++++++++----- consensus/types/src/signed_block_and_blobs.rs | 9 ++++-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index d73ec437bbe..49d655785b8 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -10,7 +10,10 @@ use slot_clock::SlotClock; use std::sync::Arc; use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; -use types::{AbstractExecPayload, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload, Hash256, SignedBeaconBlock, SignedBlockContents}; +use types::{ + AbstractExecPayload, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload, + Hash256, SignedBeaconBlock, SignedBlockContents, +}; use warp::Rejection; /// Handles a request from the HTTP API for full blocks. @@ -22,7 +25,7 @@ pub async fn publish_block( log: Logger, ) -> Result<(), Rejection> { let seen_timestamp = timestamp_now(); - let (block, _maybe_blobs) = block_contents.deconstruct(); + let (block, maybe_blobs) = block_contents.deconstruct(); let block = Arc::new(block); //FIXME(sean) have to move this to prior to publishing because it's included in the blobs sidecar message. @@ -36,13 +39,27 @@ pub async fn publish_block( // Send the block, regardless of whether or not it is valid. The API // specification is very clear that this is the desired behaviour. - let wrapped_block: BlockWrapper = - if matches!(block.as_ref(), SignedBeaconBlock::Eip4844(_)) { - todo!("to be implemented") - } else { + let wrapped_block: BlockWrapper = match block.as_ref() { + SignedBeaconBlock::Base(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Merge(_) + | SignedBeaconBlock::Capella(_) => { crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; block.into() - }; + } + SignedBeaconBlock::Eip4844(_) => { + crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; + if let Some(blobs) = maybe_blobs { + for (blob_index, blob) in blobs.into_iter().enumerate() { + crate::publish_pubsub_message( + network_tx, + PubsubMessage::BlobSidecar(Box::new((blob_index as u64, blob))), + )?; + } + } + block.into() + } + }; // Determine the delay after the start of the slot, register it with metrics. let block = wrapped_block.as_block(); diff --git a/consensus/types/src/signed_block_and_blobs.rs b/consensus/types/src/signed_block_and_blobs.rs index 3ab3d3dfee2..c6d154ef0f0 100644 --- a/consensus/types/src/signed_block_and_blobs.rs +++ b/consensus/types/src/signed_block_and_blobs.rs @@ -1,10 +1,13 @@ -use crate::{AbstractExecPayload, BlobsSidecar, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844, SignedBlobSidecar}; +use crate::{ + AbstractExecPayload, BlobsSidecar, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844, + SignedBlobSidecar, +}; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz::{Decode, DecodeError}; use ssz_derive::{Decode, Encode}; -use std::sync::Arc; use ssz_types::VariableList; +use std::sync::Arc; use tree_hash_derive::TreeHash; #[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, PartialEq)] @@ -41,4 +44,4 @@ impl SignedBeaconBlockAndBlobsSidecar { pub struct SignedBeaconBlockAndBlobSidecars> { pub signed_block: SignedBeaconBlock, pub signed_blob_sidecars: VariableList, ::MaxBlobsPerBlock>, -} \ No newline at end of file +} From fa9baab0f7e534a38bbee27ceafb222b4a9f3b7f Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 16 Mar 2023 01:43:31 +1100 Subject: [PATCH 17/96] Temporarily allow Rust check warnings on 4844 branch. (#4088) --- .github/workflows/test-suite.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 445f71fa096..54ee974c7fd 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -11,7 +11,8 @@ on: env: # Deny warnings in CI # Disable debug info (see https://github.com/sigp/lighthouse/issues/4005) - RUSTFLAGS: "-D warnings -C debuginfo=0" + # FIXME: temporarily allow warnings on 4844 branch. Revert to original later: RUSTFLAGS: "-D warnings -C debuginfo=0" + RUSTFLAGS: "-C debuginfo=0" # The Nightly version used for cargo-udeps, might need updating from time to time. PINNED_NIGHTLY: nightly-2022-12-15 # Prevent Github API rate limiting. From 2ef3ebbef320e7d65476d985e4a02ec6ffb2783d Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 16 Mar 2023 03:03:56 +1100 Subject: [PATCH 18/96] Update SignedBlobSidecar container (#4078) --- beacon_node/lighthouse_network/src/types/pubsub.rs | 2 +- .../network/src/beacon_processor/worker/gossip_methods.rs | 4 ++-- consensus/types/src/signed_blob.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 243e8314850..87012859afa 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -320,7 +320,7 @@ impl std::fmt::Display for PubsubMessage { PubsubMessage::BlobSidecar(data) => write!( f, "BlobSidecar: slot: {}, blob index: {}", - data.1.blob.slot, data.1.blob.index, + data.1.message.slot, data.1.message.index, ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index 0bdebd88fef..15f9c5e29e1 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -664,8 +664,8 @@ impl Worker { "peer_id" => %peer_id, "client" => %peer_client, "blob_topic" => blob_index, - "blob_index" => signed_blob.blob.index, - "blob_slot" => signed_blob.blob.slot + "blob_index" => signed_blob.message.index, + "blob_slot" => signed_blob.message.slot ); } diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index a3cfb9d58af..8e2e47676b7 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -19,6 +19,6 @@ use tree_hash_derive::TreeHash; #[serde(bound = "T: EthSpec")] #[arbitrary(bound = "T: EthSpec")] pub struct SignedBlobSidecar { - pub blob: BlobSidecar, + pub message: BlobSidecar, pub signature: Signature, } From 2c9477de43e4ecc011158d1cfb3a59707f7b6100 Mon Sep 17 00:00:00 2001 From: Divma <26765164+divagant-martian@users.noreply.github.com> Date: Wed, 15 Mar 2023 11:04:45 -0500 Subject: [PATCH 19/96] Fix block and blob coupling in the network context (#4086) * update docs * introduce a temp enum to model an adjusted `BlockWrapper` and fix blob coupling * fix compilation issue * fix blob coupling in the network context * review comments --- .../src/sync/block_sidecar_coupling.rs | 52 ++++++++------- beacon_node/network/src/sync/manager.rs | 4 +- .../network/src/sync/network_context.rs | 64 ++++++++++++++----- 3 files changed, 79 insertions(+), 41 deletions(-) diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index 886c9039706..438317d1cda 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -1,14 +1,13 @@ -use beacon_chain::blob_verification::BlockWrapper; +use super::network_context::TempBlockWrapper; use std::{collections::VecDeque, sync::Arc}; - -use types::{BlobsSidecar, EthSpec, SignedBeaconBlock}; +use types::{BlobSidecar, EthSpec, SignedBeaconBlock}; #[derive(Debug, Default)] pub struct BlocksAndBlobsRequestInfo { /// Blocks we have received awaiting for their corresponding sidecar. accumulated_blocks: VecDeque>>, /// Sidecars we have received awaiting for their corresponding block. - accumulated_sidecars: VecDeque>>, + accumulated_sidecars: VecDeque>>, /// Whether the individual RPC request for blocks is finished or not. is_blocks_stream_terminated: bool, /// Whether the individual RPC request for sidecars is finished or not. @@ -23,14 +22,14 @@ impl BlocksAndBlobsRequestInfo { } } - pub fn add_sidecar_response(&mut self, maybe_sidecar: Option>>) { + pub fn add_sidecar_response(&mut self, maybe_sidecar: Option>>) { match maybe_sidecar { Some(sidecar) => self.accumulated_sidecars.push_back(sidecar), None => self.is_sidecars_stream_terminated = true, } } - pub fn into_responses(self) -> Result>, &'static str> { + pub fn into_responses(self) -> Result>, &'static str> { let BlocksAndBlobsRequestInfo { accumulated_blocks, mut accumulated_sidecars, @@ -39,28 +38,33 @@ impl BlocksAndBlobsRequestInfo { // ASSUMPTION: There can't be more more blobs than blocks. i.e. sending any blob (empty // included) for a skipped slot is not permitted. - let pairs = accumulated_blocks - .into_iter() - .map(|beacon_block| { - if accumulated_sidecars - .front() - .map(|sidecar| sidecar.beacon_block_slot == beacon_block.slot()) - .unwrap_or(false) - { - let blobs_sidecar = accumulated_sidecars.pop_front(); - BlockWrapper::new(beacon_block, blobs_sidecar) - } else { - BlockWrapper::new(beacon_block, None) - } - }) - .collect::>(); + let mut responses = Vec::with_capacity(accumulated_blocks.len()); + let mut blob_iter = accumulated_sidecars.into_iter().peekable(); + for block in accumulated_blocks.into_iter() { + let mut blob_list = Vec::with_capacity(T::max_blobs_per_block()); + while { + let pair_next_blob = blob_iter + .peek() + .map(|sidecar| sidecar.slot == block.slot()) + .unwrap_or(false); + pair_next_blob + } { + blob_list.push(blob_iter.next().expect("iterator is not empty")); + } + + if blob_list.is_empty() { + responses.push(TempBlockWrapper::Block(block)) + } else { + responses.push(TempBlockWrapper::BlockAndBlobList(block, blob_list)) + } + } // if accumulated sidecars is not empty, throw an error. - if !accumulated_sidecars.is_empty() { - return Err("Received more sidecars than blocks"); + if blob_iter.next().is_some() { + return Err("Received sidecars that don't pair well"); } - Ok(pairs) + Ok(responses) } pub fn is_finished(&self) -> bool { diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 2c8e0f047fc..353d3e896ed 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -78,11 +78,11 @@ pub enum RequestId { ParentLookup { id: Id }, /// Request was from the backfill sync algorithm. BackFillBlocks { id: Id }, - /// Backfill request for blob sidecars. + /// Backfill request that is composed by both a block range request and a blob range request. BackFillBlobs { id: Id }, /// The request was from a chain in the range sync algorithm. RangeBlocks { id: Id }, - /// The request was from a chain in range, asking for ranges blob sidecars. + /// Range request that is composed by both a block range request and a blob range request. RangeBlobs { id: Id }, } diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 2da6a41e240..10f7f329557 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -18,7 +18,13 @@ use slog::{debug, trace, warn}; use std::collections::hash_map::Entry; use std::sync::Arc; use tokio::sync::mpsc; -use types::{BlobsSidecar, EthSpec, SignedBeaconBlock}; +use types::{BlobSidecar, EthSpec, SignedBeaconBlock}; + +// Temporary struct to handle incremental changes in the meantime. +pub enum TempBlockWrapper { + Block(Arc>), + BlockAndBlobList(Arc>, Vec>>), +} pub struct BlocksAndBlobsByRangeResponse { pub batch_id: BatchId, @@ -71,7 +77,7 @@ pub struct SyncNetworkContext { /// Small enumeration to make dealing with block and blob requests easier. pub enum BlockOrBlobs { Block(Option>>), - Blobs(Option>>), + Blobs(Option>>), } impl From>>> for BlockOrBlobs { @@ -80,8 +86,8 @@ impl From>>> for BlockOrBlobs { } } -impl From>>> for BlockOrBlobs { - fn from(blob: Option>>) -> Self { +impl From>>> for BlockOrBlobs { + fn from(blob: Option>>) -> Self { BlockOrBlobs::Blobs(blob) } } @@ -323,13 +329,25 @@ impl SyncNetworkContext { block_blob_info, } = entry.remove(); - Some(( - chain_id, - BlocksAndBlobsByRangeResponse { - batch_id, - responses: block_blob_info.into_responses(), - }, - )) + let responses = block_blob_info.into_responses(); + let unimplemented_info = match responses { + Ok(responses) => { + let infos = responses + .into_iter() + .map(|temp_block_wrapper| match temp_block_wrapper { + TempBlockWrapper::Block(block) => { + format!("slot{}", block.slot()) + } + TempBlockWrapper::BlockAndBlobList(block, blob_list) => { + format!("slot{}({} blobs)", block.slot(), blob_list.len()) + } + }) + .collect::>(); + infos.join(", ") + } + Err(e) => format!("Error: {e}"), + }; + unimplemented!("Here we are supposed to return a block possibly paired with a Bundle of blobs, but only have a list of individual blobs. This is what we got from the network: ChainId[{chain_id}] BatchId[{batch_id}] {unimplemented_info}") } else { None } @@ -396,10 +414,26 @@ impl SyncNetworkContext { if info.is_finished() { // If the request is finished, dequeue everything let (batch_id, info) = entry.remove(); - Some(BlocksAndBlobsByRangeResponse { - batch_id, - responses: info.into_responses(), - }) + + let responses = info.into_responses(); + let unimplemented_info = match responses { + Ok(responses) => { + let infos = responses + .into_iter() + .map(|temp_block_wrapper| match temp_block_wrapper { + TempBlockWrapper::Block(block) => { + format!("slot{}", block.slot()) + } + TempBlockWrapper::BlockAndBlobList(block, blob_list) => { + format!("slot{}({} blobs)", block.slot(), blob_list.len()) + } + }) + .collect::>(); + infos.join(", ") + } + Err(e) => format!("Error: {e}"), + }; + unimplemented!("Here we are supposed to return a block possibly paired with a Bundle of blobs for backfill, but only have a list of individual blobs. This is what we got from the network: BatchId[{batch_id}]{unimplemented_info}") } else { None } From 3d99e1f14dc1b9050d619bbcae154d96fb184f5f Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 Mar 2023 12:15:08 -0400 Subject: [PATCH 20/96] move block contents to api crate, rename blob sidecars list --- beacon_node/beacon_chain/src/blob_cache.rs | 12 ++-- .../http_api/src/build_block_contents.rs | 5 +- common/eth2/src/types.rs | 60 +++++++++++++++++++ consensus/types/src/beacon_block.rs | 11 ---- .../src/beacon_block_and_blob_sidecars.rs | 6 +- consensus/types/src/blob_sidecar.rs | 2 +- consensus/types/src/block_contents.rs | 56 ----------------- consensus/types/src/lib.rs | 4 +- 8 files changed, 75 insertions(+), 81 deletions(-) delete mode 100644 consensus/types/src/block_contents.rs diff --git a/beacon_node/beacon_chain/src/blob_cache.rs b/beacon_node/beacon_chain/src/blob_cache.rs index e3b0a06b64a..64f113c285c 100644 --- a/beacon_node/beacon_chain/src/blob_cache.rs +++ b/beacon_node/beacon_chain/src/blob_cache.rs @@ -1,12 +1,12 @@ use lru::LruCache; use parking_lot::Mutex; -use types::{BlobSidecars, EthSpec, Hash256}; +use types::{BlobSidecarList, EthSpec, Hash256}; pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10; /// A cache blobs by beacon block root. pub struct BlobCache { - blobs: Mutex>>, + blobs: Mutex>>, } #[derive(Hash, PartialEq, Eq)] @@ -21,11 +21,15 @@ impl Default for BlobCache { } impl BlobCache { - pub fn put(&self, beacon_block: Hash256, blobs: BlobSidecars) -> Option> { + pub fn put( + &self, + beacon_block: Hash256, + blobs: BlobSidecarList, + ) -> Option> { self.blobs.lock().put(BlobCacheId(beacon_block), blobs) } - pub fn pop(&self, root: &Hash256) -> Option> { + pub fn pop(&self, root: &Hash256) -> Option> { self.blobs.lock().pop(&BlobCacheId(*root)) } } diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index d456c320a63..1908c03ea1c 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,8 +1,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProductionError}; +use eth2::types::BlockContents; use std::sync::Arc; -use types::{ - AbstractExecPayload, BeaconBlock, BeaconBlockAndBlobSidecars, BlockContents, ForkName, -}; +use types::{AbstractExecPayload, BeaconBlock, BeaconBlockAndBlobSidecars, ForkName}; type Error = warp::reject::Rejection; diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index d6db3e15ff6..10c5b86f6f6 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1259,3 +1259,63 @@ mod tests { ) } } + +/// A wrapper over a [`BeaconBlock`] or a [`BeaconBlockAndBlobSidecars`]. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +#[serde(bound = "T: EthSpec")] +pub enum BlockContents> { + BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), + Block(BeaconBlock), +} + +impl> BlockContents { + pub fn block(&self) -> &BeaconBlock { + match self { + BlockContents::BlockAndBlobSidecars(block_and_sidecars) => &block_and_sidecars.block, + BlockContents::Block(block) => block, + } + } + + pub fn deconstruct(self) -> (BeaconBlock, Option>) { + match self { + BlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( + block_and_sidecars.block, + Some(block_and_sidecars.blob_sidecars), + ), + BlockContents::Block(block) => (block, None), + } + } +} + +impl> ForkVersionDeserialize + for BlockContents +{ + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::value::Value, + fork_name: ForkName, + ) -> Result { + match fork_name { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + Ok(BlockContents::Block(BeaconBlock::deserialize_by_fork::< + 'de, + D, + >(value, fork_name)?)) + } + ForkName::Eip4844 => Ok(BlockContents::BlockAndBlobSidecars( + BeaconBlockAndBlobSidecars::deserialize_by_fork::<'de, D>(value, fork_name)?, + )), + } + } +} + +impl> Into> + for BlockContents +{ + fn into(self) -> BeaconBlock { + match self { + Self::BlockAndBlobSidecars(block_and_sidecars) => block_and_sidecars.block, + Self::Block(block) => block, + } + } +} diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index a929265824a..0f26cd0e5e7 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -731,17 +731,6 @@ impl From>> } } -impl> From> - for BeaconBlock -{ - fn from(block_contents: BlockContents) -> Self { - match block_contents { - BlockContents::BlockAndBlobSidecars(block_and_sidecars) => block_and_sidecars.block, - BlockContents::Block(block) => block, - } - } -} - impl> ForkVersionDeserialize for BeaconBlock { diff --git a/consensus/types/src/beacon_block_and_blob_sidecars.rs b/consensus/types/src/beacon_block_and_blob_sidecars.rs index c518f765b18..78e70419614 100644 --- a/consensus/types/src/beacon_block_and_blob_sidecars.rs +++ b/consensus/types/src/beacon_block_and_blob_sidecars.rs @@ -1,5 +1,5 @@ use crate::{ - AbstractExecPayload, BeaconBlock, BlobSidecars, EthSpec, ForkName, ForkVersionDeserialize, + AbstractExecPayload, BeaconBlock, BlobSidecarList, EthSpec, ForkName, ForkVersionDeserialize, }; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; @@ -11,7 +11,7 @@ use tree_hash_derive::TreeHash; #[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] pub struct BeaconBlockAndBlobSidecars> { pub block: BeaconBlock, - pub blob_sidecars: BlobSidecars, + pub blob_sidecars: BlobSidecarList, } impl> ForkVersionDeserialize @@ -25,7 +25,7 @@ impl> ForkVersionDeserialize #[serde(bound = "T: EthSpec")] struct Helper { block: serde_json::Value, - blob_sidecars: BlobSidecars, + blob_sidecars: BlobSidecarList, } let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 251bef03436..169c570d291 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -47,7 +47,7 @@ pub struct BlobSidecar { pub kzg_proof: KzgProof, } -pub type BlobSidecars = VariableList, ::MaxBlobsPerBlock>; +pub type BlobSidecarList = VariableList, ::MaxBlobsPerBlock>; impl SignedRoot for BlobSidecar {} diff --git a/consensus/types/src/block_contents.rs b/consensus/types/src/block_contents.rs deleted file mode 100644 index d5a500280c9..00000000000 --- a/consensus/types/src/block_contents.rs +++ /dev/null @@ -1,56 +0,0 @@ -use crate::{ - AbstractExecPayload, BeaconBlock, BeaconBlockAndBlobSidecars, BlobSidecars, EthSpec, ForkName, - ForkVersionDeserialize, -}; -use derivative::Derivative; -use serde_derive::{Deserialize, Serialize}; - -/// A wrapper over a [`BeaconBlock`] or a [`BeaconBlockAndBlobSidecars`]. -#[derive(Clone, Debug, Derivative, Serialize, Deserialize)] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] -#[serde(untagged)] -#[serde(bound = "T: EthSpec")] -pub enum BlockContents> { - BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), - Block(BeaconBlock), -} - -impl> BlockContents { - pub fn block(&self) -> &BeaconBlock { - match self { - BlockContents::BlockAndBlobSidecars(block_and_sidecars) => &block_and_sidecars.block, - BlockContents::Block(block) => block, - } - } - - pub fn deconstruct(self) -> (BeaconBlock, Option>) { - match self { - BlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( - block_and_sidecars.block, - Some(block_and_sidecars.blob_sidecars), - ), - BlockContents::Block(block) => (block, None), - } - } -} - -impl> ForkVersionDeserialize - for BlockContents -{ - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { - match fork_name { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { - Ok(BlockContents::Block(BeaconBlock::deserialize_by_fork::< - 'de, - D, - >(value, fork_name)?)) - } - ForkName::Eip4844 => Ok(BlockContents::BlockAndBlobSidecars( - BeaconBlockAndBlobSidecars::deserialize_by_fork::<'de, D>(value, fork_name)?, - )), - } - } -} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 50b9547c9c4..fe47454aa2a 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -102,7 +102,6 @@ pub mod sqlite; pub mod beacon_block_and_blob_sidecars; pub mod blob_sidecar; pub mod blobs_sidecar; -pub mod block_contents; pub mod signed_blob; pub mod signed_block_and_blobs; pub mod transaction; @@ -126,9 +125,8 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; -pub use crate::blob_sidecar::{BlobSidecar, BlobSidecars}; +pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList}; pub use crate::blobs_sidecar::{Blobs, BlobsSidecar}; -pub use crate::block_contents::BlockContents; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; From 34cea6d1c3b1bcec6bbe9a635009a0bb3af18a39 Mon Sep 17 00:00:00 2001 From: Diva M Date: Wed, 15 Mar 2023 12:37:53 -0500 Subject: [PATCH 21/96] fmt --- beacon_node/http_api/src/lib.rs | 4 ++-- consensus/types/src/lib.rs | 7 +++++-- consensus/types/src/signed_blob.rs | 2 +- consensus/types/src/signed_block_contents.rs | 19 ++++++++++++++----- validator_client/src/block_service.rs | 5 ++++- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 04cb11cd06c..7ac23a9d905 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -57,9 +57,9 @@ use types::{ Attestation, AttestationData, AttesterSlashing, BeaconStateError, BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, FullPayload, ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, - SignedBeaconBlock, SignedBlindedBeaconBlock, SignedBlsToExecutionChange, + SignedBeaconBlock, SignedBlindedBeaconBlock, SignedBlockContents, SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, - SyncCommitteeMessage, SyncContributionData, SignedBlockContents, + SyncCommitteeMessage, SyncContributionData, }; use version::{ add_consensus_version_header, execution_optimistic_fork_versioned_response, diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 1c6f0b9efc7..14f06bb51d6 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -102,8 +102,8 @@ pub mod blob_sidecar; pub mod blobs_sidecar; pub mod signed_blob; pub mod signed_block_and_blobs; -pub mod transaction; pub mod signed_block_contents; +pub mod transaction; use ethereum_types::{H160, H256}; @@ -184,7 +184,10 @@ pub use crate::signed_beacon_block::{ }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_blob::*; -pub use crate::signed_block_and_blobs::{SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockAndBlobsSidecarDecode, SignedBeaconBlockAndBlobSidecars}; +pub use crate::signed_block_and_blobs::{ + SignedBeaconBlockAndBlobSidecars, SignedBeaconBlockAndBlobsSidecar, + SignedBeaconBlockAndBlobsSidecarDecode, +}; pub use crate::signed_block_contents::SignedBlockContents; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; pub use crate::signed_contribution_and_proof::SignedContributionAndProof; diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index d7d2f884d49..4121b8b7f29 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -1,9 +1,9 @@ use crate::{test_utils::TestRandom, BlobSidecar, EthSpec, Signature}; +use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; -use derivative::Derivative; #[derive( Debug, diff --git a/consensus/types/src/signed_block_contents.rs b/consensus/types/src/signed_block_contents.rs index a6743c60f17..bce62333383 100644 --- a/consensus/types/src/signed_block_contents.rs +++ b/consensus/types/src/signed_block_contents.rs @@ -1,5 +1,5 @@ -use crate::{AbstractExecPayload, EthSpec, FullPayload, SignedBeaconBlock, SignedBlobSidecar}; use crate::signed_block_and_blobs::SignedBeaconBlockAndBlobSidecars; +use crate::{AbstractExecPayload, EthSpec, FullPayload, SignedBeaconBlock, SignedBlobSidecar}; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_types::VariableList; @@ -17,12 +17,19 @@ pub enum SignedBlockContents = FullP impl> SignedBlockContents { pub fn signed_block(&self) -> &SignedBeaconBlock { match self { - SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => &block_and_sidecars.signed_block, + SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { + &block_and_sidecars.signed_block + } SignedBlockContents::Block(block) => block, } } - pub fn deconstruct(self) -> (SignedBeaconBlock, Option, ::MaxBlobsPerBlock>>) { + pub fn deconstruct( + self, + ) -> ( + SignedBeaconBlock, + Option, ::MaxBlobsPerBlock>>, + ) { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( block_and_sidecars.signed_block, @@ -33,8 +40,10 @@ impl> SignedBlockContents> From> for SignedBlockContents { +impl> From> + for SignedBlockContents +{ fn from(block: SignedBeaconBlock) -> Self { SignedBlockContents::Block(block) } -} \ No newline at end of file +} diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 0cc0e40a1a8..eb40dee9b33 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -14,7 +14,10 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use tokio::time::sleep; -use types::{AbstractExecPayload, BeaconBlock, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, PublicKeyBytes, SignedBlockContents, Slot}; +use types::{ + AbstractExecPayload, BeaconBlock, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, + PublicKeyBytes, SignedBlockContents, Slot, +}; #[derive(Debug)] pub enum BlockError { From b303d2fb7eadf8fa471ad546f4b2fd45d09f6e2c Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 Mar 2023 15:32:22 -0400 Subject: [PATCH 22/96] lints --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- beacon_node/beacon_chain/src/blob_verification.rs | 10 +++++----- beacon_node/http_api/src/build_block_contents.rs | 4 ++-- .../src/beacon_processor/worker/gossip_methods.rs | 4 ++-- .../src/beacon_processor/worker/rpc_methods.rs | 4 ++-- .../network/src/sync/block_sidecar_coupling.rs | 2 +- beacon_node/network/src/sync/manager.rs | 6 +++--- consensus/types/src/signed_block_contents.rs | 12 ++++++------ validator_client/src/block_service.rs | 3 +-- 9 files changed, 23 insertions(+), 24 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 8f39b3758df..79a3e418e84 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4857,7 +4857,7 @@ impl BeaconChain { )), )?; - kzg_utils::compute_blob_kzg_proof::(kzg, blob, kzg_commitment.clone()) + kzg_utils::compute_blob_kzg_proof::(kzg, blob, *kzg_commitment) .map_err(BlockProductionError::KzgError) }) .collect::, BlockProductionError>>() diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 7a4493c9736..e2288dbb808 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -3,7 +3,7 @@ use slot_clock::SlotClock; use std::sync::Arc; use crate::beacon_chain::{BeaconChain, BeaconChainTypes, MAXIMUM_GOSSIP_CLOCK_DISPARITY}; -use crate::{kzg_utils, BeaconChainError}; +use crate::BeaconChainError; use state_processing::per_block_processing::eip4844::eip4844::verify_kzg_commitments_against_transactions; use types::signed_beacon_block::BlobReconstructionError; use types::{ @@ -116,11 +116,11 @@ pub fn validate_blob_for_gossip( } fn verify_data_availability( - blob_sidecar: &BlobsSidecar, + _blob_sidecar: &BlobsSidecar, kzg_commitments: &[KzgCommitment], transactions: &Transactions, - block_slot: Slot, - block_root: Hash256, + _block_slot: Slot, + _block_root: Hash256, chain: &BeaconChain, ) -> Result<(), BlobError> { if verify_kzg_commitments_against_transactions::(transactions, kzg_commitments) @@ -130,7 +130,7 @@ fn verify_data_availability( } // Validatate that the kzg proof is valid against the commitments and blobs - let kzg = chain + let _kzg = chain .kzg .as_ref() .ok_or(BlobError::TrustedSetupNotInitialized)?; diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index 1908c03ea1c..9fbde0ce06a 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -24,9 +24,9 @@ pub fn build_block_contents Worker { #[allow(clippy::too_many_arguments)] pub async fn process_gossip_blob( self, - message_id: MessageId, + _message_id: MessageId, peer_id: PeerId, peer_client: Client, blob_index: u64, signed_blob: Arc>, - seen_duration: Duration, + _seen_duration: Duration, ) { // TODO: gossip verification crit!(self.log, "UNIMPLEMENTED gossip blob verification"; diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 4480f37130e..78b9de303fc 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -252,7 +252,7 @@ impl Worker { block_parent_root: block.parent_root, proposer_index: block.proposer_index, blob, - kzg_commitment: block.body.blob_kzg_commitments[known_index].clone(), // TODO: needs to be stored in a more logical way so that this won't panic. + kzg_commitment: block.body.blob_kzg_commitments[known_index], // TODO: needs to be stored in a more logical way so that this won't panic. kzg_proof: kzg_aggregated_proof // TODO: yeah }; self.send_response( @@ -843,7 +843,7 @@ impl Worker { beacon_block_root, beacon_block_slot, blobs: blob_bundle, - kzg_aggregated_proof, + kzg_aggregated_proof: _, }: types::BlobsSidecar<_> = blobs; for (blob_index, blob) in blob_bundle.into_iter().enumerate() { diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index 438317d1cda..67db9a7a326 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -32,7 +32,7 @@ impl BlocksAndBlobsRequestInfo { pub fn into_responses(self) -> Result>, &'static str> { let BlocksAndBlobsRequestInfo { accumulated_blocks, - mut accumulated_sidecars, + accumulated_sidecars, .. } = self; diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 353d3e896ed..768b95273ed 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -875,8 +875,8 @@ impl SyncManager { fn rpc_blobs_received( &mut self, request_id: RequestId, - peer_id: PeerId, - maybe_blob: Option::EthSpec>>>, + _peer_id: PeerId, + _maybe_blob: Option::EthSpec>>>, _seen_timestamp: Duration, ) { match request_id { @@ -892,7 +892,7 @@ impl SyncManager { RequestId::RangeBlocks { .. } => { unreachable!("Only-blocks range requests don't receive sidecars") } - RequestId::RangeBlobs { id } => { + RequestId::RangeBlobs { id: _ } => { unimplemented!("Adjust range"); } } diff --git a/consensus/types/src/signed_block_contents.rs b/consensus/types/src/signed_block_contents.rs index bce62333383..7f547c86fa8 100644 --- a/consensus/types/src/signed_block_contents.rs +++ b/consensus/types/src/signed_block_contents.rs @@ -4,6 +4,11 @@ use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_types::VariableList; +pub type BlockContentsTuple = ( + SignedBeaconBlock, + Option, ::MaxBlobsPerBlock>>, +); + /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. #[derive(Clone, Debug, Derivative, Serialize, Deserialize)] #[derivative(PartialEq, Hash(bound = "T: EthSpec"))] @@ -24,12 +29,7 @@ impl> SignedBlockContents ( - SignedBeaconBlock, - Option, ::MaxBlobsPerBlock>>, - ) { + pub fn deconstruct(self) -> BlockContentsTuple { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( block_and_sidecars.signed_block, diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index eb40dee9b33..48d64601680 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -388,7 +388,6 @@ impl BlockService { )) })? .data - .into() } }; @@ -455,7 +454,7 @@ impl BlockService { ); beacon_node // TODO: need to be adjusted for blobs - .post_beacon_blinded_blocks(&signed_block_contents.signed_block()) + .post_beacon_blinded_blocks(signed_block_contents.signed_block()) .await .map_err(|e| { BlockError::Irrecoverable(format!( From fb7d729d92d500c04243038af8c6a073afdd280f Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 Mar 2023 16:03:36 -0400 Subject: [PATCH 23/96] migrate types to API crate --- beacon_node/http_api/src/lib.rs | 5 +- beacon_node/http_api/src/publish_blocks.rs | 3 +- .../http_api/tests/interactive_tests.rs | 7 ++- beacon_node/http_api/tests/tests.rs | 4 +- common/eth2/src/types.rs | 51 +++++++++++++++++++ consensus/types/src/lib.rs | 5 +- consensus/types/src/signed_block_and_blobs.rs | 14 +---- consensus/types/src/signed_block_contents.rs | 49 ------------------ validator_client/src/block_service.rs | 3 +- 9 files changed, 68 insertions(+), 73 deletions(-) delete mode 100644 consensus/types/src/signed_block_contents.rs diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 7ac23a9d905..e48f8d7ad1c 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -31,7 +31,8 @@ use beacon_chain::{ pub use block_id::BlockId; use directory::DEFAULT_ROOT_DIR; use eth2::types::{ - self as api_types, EndpointVersion, SkipRandaoVerification, ValidatorId, ValidatorStatus, + self as api_types, EndpointVersion, SignedBlockContents, SkipRandaoVerification, ValidatorId, + ValidatorStatus, }; use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage}; use lighthouse_version::version_with_platform; @@ -57,7 +58,7 @@ use types::{ Attestation, AttestationData, AttesterSlashing, BeaconStateError, BlindedPayload, CommitteeCache, ConfigAndPreset, Epoch, EthSpec, ForkName, FullPayload, ProposerPreparationData, ProposerSlashing, RelativeEpoch, SignedAggregateAndProof, - SignedBeaconBlock, SignedBlindedBeaconBlock, SignedBlockContents, SignedBlsToExecutionChange, + SignedBeaconBlock, SignedBlindedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData, }; diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 49d655785b8..dc8bb020ac7 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -3,6 +3,7 @@ use beacon_chain::blob_verification::{AsBlock, BlockWrapper, IntoAvailableBlock} use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::NotifyExecutionLayer; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, CountUnrealized}; +use eth2::types::SignedBlockContents; use lighthouse_network::PubsubMessage; use network::NetworkMessage; use slog::{debug, error, info, warn, Logger}; @@ -12,7 +13,7 @@ use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ AbstractExecPayload, BlindedPayload, EthSpec, ExecPayload, ExecutionBlockHash, FullPayload, - Hash256, SignedBeaconBlock, SignedBlockContents, + Hash256, SignedBeaconBlock, }; use warp::Rejection; diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 7db1b22d67e..00fa7faff0d 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -513,12 +513,13 @@ pub async fn proposer_boost_re_org_test( let randao_reveal = harness .sign_randao_reveal(&state_b, proposer_index, slot_c) .into(); - let unsigned_block_c = tester + let unsigned_block_contents_c = tester .client .get_validator_blocks(slot_c, &randao_reveal, None) .await .unwrap() .data; + let unsigned_block_c = unsigned_block_contents_c.deconstruct().0; let block_c = harness.sign_beacon_block(unsigned_block_c, &state_b); if should_re_org { @@ -700,7 +701,9 @@ pub async fn fork_choice_before_proposal() { .get_validator_blocks::>(slot_d, &randao_reveal, None) .await .unwrap() - .data; + .data + .deconstruct() + .0; // Head is now B. assert_eq!( diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 977c737fd0b..423e2d4de5f 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -2065,7 +2065,9 @@ impl ApiTester { .get_validator_blocks::>(slot, &randao_reveal, None) .await .unwrap() - .data; + .data + .deconstruct() + .0; let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index d328639120e..ffd7a60e4db 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -5,6 +5,7 @@ use crate::Error as ServerError; use lighthouse_network::{ConnectionDirection, Enr, Multiaddr, PeerConnectionStatus}; use mime::{Mime, APPLICATION, JSON, OCTET_STREAM, STAR}; use serde::{Deserialize, Serialize}; +use ssz_derive::Encode; use std::cmp::Reverse; use std::convert::TryFrom; use std::fmt; @@ -1322,3 +1323,53 @@ impl> Into> } } } + +pub type BlockContentsTuple = ( + SignedBeaconBlock, + Option, ::MaxBlobsPerBlock>>, +); + +/// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +#[serde(bound = "T: EthSpec")] +pub enum SignedBlockContents = FullPayload> { + BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars), + Block(SignedBeaconBlock), +} + +impl> SignedBlockContents { + pub fn signed_block(&self) -> &SignedBeaconBlock { + match self { + SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { + &block_and_sidecars.signed_block + } + SignedBlockContents::Block(block) => block, + } + } + + pub fn deconstruct(self) -> BlockContentsTuple { + match self { + SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( + block_and_sidecars.signed_block, + Some(block_and_sidecars.signed_blob_sidecars), + ), + SignedBlockContents::Block(block) => (block, None), + } + } +} + +impl> From> + for SignedBlockContents +{ + fn from(block: SignedBeaconBlock) -> Self { + SignedBlockContents::Block(block) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +#[serde(bound = "T: EthSpec")] +pub struct SignedBeaconBlockAndBlobSidecars> { + pub signed_block: SignedBeaconBlock, + pub signed_blob_sidecars: VariableList, ::MaxBlobsPerBlock>, +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 14f06bb51d6..6a86e773a1b 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -102,7 +102,6 @@ pub mod blob_sidecar; pub mod blobs_sidecar; pub mod signed_blob; pub mod signed_block_and_blobs; -pub mod signed_block_contents; pub mod transaction; use ethereum_types::{H160, H256}; @@ -185,10 +184,8 @@ pub use crate::signed_beacon_block::{ pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_blob::*; pub use crate::signed_block_and_blobs::{ - SignedBeaconBlockAndBlobSidecars, SignedBeaconBlockAndBlobsSidecar, - SignedBeaconBlockAndBlobsSidecarDecode, + SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockAndBlobsSidecarDecode, }; -pub use crate::signed_block_contents::SignedBlockContents; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; pub use crate::signed_contribution_and_proof::SignedContributionAndProof; pub use crate::signed_voluntary_exit::SignedVoluntaryExit; diff --git a/consensus/types/src/signed_block_and_blobs.rs b/consensus/types/src/signed_block_and_blobs.rs index c6d154ef0f0..a3bd34475d7 100644 --- a/consensus/types/src/signed_block_and_blobs.rs +++ b/consensus/types/src/signed_block_and_blobs.rs @@ -1,12 +1,8 @@ -use crate::{ - AbstractExecPayload, BlobsSidecar, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844, - SignedBlobSidecar, -}; +use crate::{BlobsSidecar, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844}; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz::{Decode, DecodeError}; use ssz_derive::{Decode, Encode}; -use ssz_types::VariableList; use std::sync::Arc; use tree_hash_derive::TreeHash; @@ -37,11 +33,3 @@ impl SignedBeaconBlockAndBlobsSidecar { }) } } - -#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] -#[serde(bound = "T: EthSpec")] -pub struct SignedBeaconBlockAndBlobSidecars> { - pub signed_block: SignedBeaconBlock, - pub signed_blob_sidecars: VariableList, ::MaxBlobsPerBlock>, -} diff --git a/consensus/types/src/signed_block_contents.rs b/consensus/types/src/signed_block_contents.rs deleted file mode 100644 index 7f547c86fa8..00000000000 --- a/consensus/types/src/signed_block_contents.rs +++ /dev/null @@ -1,49 +0,0 @@ -use crate::signed_block_and_blobs::SignedBeaconBlockAndBlobSidecars; -use crate::{AbstractExecPayload, EthSpec, FullPayload, SignedBeaconBlock, SignedBlobSidecar}; -use derivative::Derivative; -use serde_derive::{Deserialize, Serialize}; -use ssz_types::VariableList; - -pub type BlockContentsTuple = ( - SignedBeaconBlock, - Option, ::MaxBlobsPerBlock>>, -); - -/// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. -#[derive(Clone, Debug, Derivative, Serialize, Deserialize)] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] -#[serde(untagged)] -#[serde(bound = "T: EthSpec")] -pub enum SignedBlockContents = FullPayload> { - BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars), - Block(SignedBeaconBlock), -} - -impl> SignedBlockContents { - pub fn signed_block(&self) -> &SignedBeaconBlock { - match self { - SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { - &block_and_sidecars.signed_block - } - SignedBlockContents::Block(block) => block, - } - } - - pub fn deconstruct(self) -> BlockContentsTuple { - match self { - SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( - block_and_sidecars.signed_block, - Some(block_and_sidecars.signed_blob_sidecars), - ), - SignedBlockContents::Block(block) => (block, None), - } - } -} - -impl> From> - for SignedBlockContents -{ - fn from(block: SignedBeaconBlock) -> Self { - SignedBlockContents::Block(block) - } -} diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 48d64601680..0eb9a07c394 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -7,6 +7,7 @@ use crate::{ }; use crate::{http_metrics::metrics, validator_store::ValidatorStore}; use environment::RuntimeContext; +use eth2::types::SignedBlockContents; use slog::{crit, debug, error, info, trace, warn}; use slot_clock::SlotClock; use std::ops::Deref; @@ -16,7 +17,7 @@ use tokio::sync::mpsc; use tokio::time::sleep; use types::{ AbstractExecPayload, BeaconBlock, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, - PublicKeyBytes, SignedBlockContents, Slot, + PublicKeyBytes, Slot, }; #[derive(Debug)] From cf4285e1d4321f76ae0142fed2df0feeead5605a Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 15 Mar 2023 16:34:00 -0400 Subject: [PATCH 24/96] compile tests --- beacon_node/http_api/tests/tests.rs | 58 +++++++++++++++++++---------- common/eth2/src/types.rs | 9 ++++- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 423e2d4de5f..dae17006bc0 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -61,8 +61,8 @@ struct ApiTester { harness: Arc>>, chain: Arc>>, client: BeaconNodeHttpClient, - next_block: SignedBeaconBlock, - reorg_block: SignedBeaconBlock, + next_block: SignedBlockContents, + reorg_block: SignedBlockContents, attestations: Vec>, contribution_and_proofs: Vec>, attester_slashing: AttesterSlashing, @@ -154,11 +154,13 @@ impl ApiTester { let (next_block, _next_state) = harness .make_block(head.beacon_state.clone(), harness.chain.slot().unwrap()) .await; + let next_block = SignedBlockContents::from(next_block); // `make_block` adds random graffiti, so this will produce an alternate block let (reorg_block, _reorg_state) = harness .make_block(head.beacon_state.clone(), harness.chain.slot().unwrap()) .await; + let reorg_block = SignedBlockContents::from(reorg_block); let head_state_root = head.beacon_state_root(); let attestations = harness @@ -288,11 +290,13 @@ impl ApiTester { let (next_block, _next_state) = harness .make_block(head.beacon_state.clone(), harness.chain.slot().unwrap()) .await; + let next_block = SignedBlockContents::from(next_block); // `make_block` adds random graffiti, so this will produce an alternate block let (reorg_block, _reorg_state) = harness .make_block(head.beacon_state.clone(), harness.chain.slot().unwrap()) .await; + let reorg_block = SignedBlockContents::from(reorg_block); let head_state_root = head.beacon_state_root(); let attestations = harness @@ -975,9 +979,9 @@ impl ApiTester { } pub async fn test_post_beacon_blocks_valid(mut self) -> Self { - let next_block = &self.next_block; + let next_block = self.next_block.clone(); - self.client.post_beacon_blocks(next_block).await.unwrap(); + self.client.post_beacon_blocks(&next_block).await.unwrap(); assert!( self.network_rx.network_recv.recv().await.is_some(), @@ -988,10 +992,14 @@ impl ApiTester { } pub async fn test_post_beacon_blocks_invalid(mut self) -> Self { - let mut next_block = self.next_block.clone(); + let mut next_block = self.next_block.clone().deconstruct().0; *next_block.message_mut().proposer_index_mut() += 1; - assert!(self.client.post_beacon_blocks(&next_block).await.is_err()); + assert!(self + .client + .post_beacon_blocks(&SignedBlockContents::from(next_block)) + .await + .is_err()); assert!( self.network_rx.network_recv.recv().await.is_some(), @@ -2070,8 +2078,12 @@ impl ApiTester { .0; let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); + let signed_block_contents = SignedBlockContents::from(signed_block.clone()); - self.client.post_beacon_blocks(&signed_block).await.unwrap(); + self.client + .post_beacon_blocks(&signed_block_contents) + .await + .unwrap(); assert_eq!(self.chain.head_beacon_block().as_ref(), &signed_block); @@ -2095,7 +2107,9 @@ impl ApiTester { ) .await .unwrap() - .data; + .data + .deconstruct() + .0; assert_eq!(block.slot(), slot); self.chain.slot_clock.set_slot(slot.as_u64() + 1); } @@ -3762,12 +3776,12 @@ impl ApiTester { // Submit the next block, which is on an epoch boundary, so this will produce a finalized // checkpoint event, head event, and block event - let block_root = self.next_block.canonical_root(); + let block_root = self.next_block.signed_block().canonical_root(); // current_duty_dependent_root = block root because this is the first slot of the epoch let current_duty_dependent_root = self.chain.head_beacon_block_root(); let current_slot = self.chain.slot().unwrap(); - let next_slot = self.next_block.slot(); + let next_slot = self.next_block.signed_block().slot(); let finalization_distance = E::slots_per_epoch() * 2; let expected_block = EventKind::Block(SseBlock { @@ -3779,7 +3793,7 @@ impl ApiTester { let expected_head = EventKind::Head(SseHead { block: block_root, slot: next_slot, - state: self.next_block.state_root(), + state: self.next_block.signed_block().state_root(), current_duty_dependent_root, previous_duty_dependent_root: self .chain @@ -3828,13 +3842,17 @@ impl ApiTester { .unwrap(); let expected_reorg = EventKind::ChainReorg(SseChainReorg { - slot: self.next_block.slot(), + slot: self.next_block.signed_block().slot(), depth: 1, - old_head_block: self.next_block.canonical_root(), - old_head_state: self.next_block.state_root(), - new_head_block: self.reorg_block.canonical_root(), - new_head_state: self.reorg_block.state_root(), - epoch: self.next_block.slot().epoch(E::slots_per_epoch()), + old_head_block: self.next_block.signed_block().canonical_root(), + old_head_state: self.next_block.signed_block().state_root(), + new_head_block: self.reorg_block.signed_block().canonical_root(), + new_head_state: self.reorg_block.signed_block().state_root(), + epoch: self + .next_block + .signed_block() + .slot() + .epoch(E::slots_per_epoch()), execution_optimistic: false, }); @@ -3896,8 +3914,8 @@ impl ApiTester { .await .unwrap(); - let block_root = self.next_block.canonical_root(); - let next_slot = self.next_block.slot(); + let block_root = self.next_block.signed_block().canonical_root(); + let next_slot = self.next_block.signed_block().slot(); let expected_block = EventKind::Block(SseBlock { block: block_root, @@ -3908,7 +3926,7 @@ impl ApiTester { let expected_head = EventKind::Head(SseHead { block: block_root, slot: next_slot, - state: self.next_block.state_root(), + state: self.next_block.signed_block().state_root(), current_duty_dependent_root: self.chain.genesis_block_root, previous_duty_dependent_root: self.chain.genesis_block_root, epoch_transition: false, diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index ffd7a60e4db..db64d74c2ad 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1363,7 +1363,14 @@ impl> From { fn from(block: SignedBeaconBlock) -> Self { - SignedBlockContents::Block(block) + match block { + SignedBeaconBlock::Base(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Merge(_) + | SignedBeaconBlock::Capella(_) => SignedBlockContents::Block(block), + //TODO: error handling, this should be try from + SignedBeaconBlock::Eip4844(_block) => todo!(), + } } } From 3c18e1a3a4c0a4895dcd08e79e34b6c59420edd9 Mon Sep 17 00:00:00 2001 From: Divma <26765164+divagant-martian@users.noreply.github.com> Date: Thu, 16 Mar 2023 19:20:39 -0500 Subject: [PATCH 25/96] thread blocks and blobs to sync (#4100) * thread blocks and blobs to sync * satisfy dead code analysis --- beacon_node/network/src/router/processor.rs | 4 +- beacon_node/network/src/sync/manager.rs | 126 ++++++++++-------- .../network/src/sync/network_context.rs | 24 ++-- 3 files changed, 83 insertions(+), 71 deletions(-) diff --git a/beacon_node/network/src/router/processor.rs b/beacon_node/network/src/router/processor.rs index 56a5f245867..76962b373fe 100644 --- a/beacon_node/network/src/router/processor.rs +++ b/beacon_node/network/src/router/processor.rs @@ -258,7 +258,7 @@ impl Processor { ); if let RequestId::Sync(id) = request_id { - self.send_to_sync(SyncMessage::RpcBlobs { + self.send_to_sync(SyncMessage::RpcBlob { peer_id, request_id: id, blob_sidecar, @@ -330,7 +330,7 @@ impl Processor { "Received BlobsByRoot Response"; "peer" => %peer_id, ); - self.send_to_sync(SyncMessage::RpcBlobs { + self.send_to_sync(SyncMessage::RpcBlob { request_id, peer_id, blob_sidecar, diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 768b95273ed..43921b585a4 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -35,7 +35,7 @@ use super::backfill_sync::{BackFillSync, ProcessResult, SyncStart}; use super::block_lookups::BlockLookups; -use super::network_context::{BlockOrBlobs, SyncNetworkContext}; +use super::network_context::{BlockOrBlob, SyncNetworkContext}; use super::peer_sync_info::{remote_sync_type, PeerSyncType}; use super::range_sync::{RangeSync, RangeSyncType, EPOCHS_PER_BATCH}; use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEvent}; @@ -86,6 +86,10 @@ pub enum RequestId { RangeBlobs { id: Id }, } +// TODO(diva) I'm updating functions what at a time, but this should be revisited because I think +// some code paths that are split for blobs and blocks can be made just one after sync as a whole +// is updated. + #[derive(Debug)] /// A message that can be sent to the sync manager thread. pub enum SyncMessage { @@ -101,7 +105,7 @@ pub enum SyncMessage { }, /// A blob has been received from the RPC. - RpcBlobs { + RpcBlob { request_id: RequestId, peer_id: PeerId, blob_sidecar: Option>>, @@ -554,7 +558,12 @@ impl SyncManager { beacon_block, seen_timestamp, } => { - self.rpc_block_received(request_id, peer_id, beacon_block, seen_timestamp); + self.rpc_block_or_blob_received( + request_id, + peer_id, + beacon_block.into(), + seen_timestamp, + ); } SyncMessage::UnknownBlock(peer_id, block, block_root) => { // If we are not synced or within SLOT_IMPORT_TOLERANCE of the block, ignore @@ -638,12 +647,17 @@ impl SyncManager { .block_lookups .parent_chain_processed(chain_hash, result, &mut self.network), }, - SyncMessage::RpcBlobs { + SyncMessage::RpcBlob { request_id, peer_id, blob_sidecar, seen_timestamp, - } => self.rpc_blobs_received(request_id, peer_id, blob_sidecar, seen_timestamp), + } => self.rpc_block_or_blob_received( + request_id, + peer_id, + blob_sidecar.into(), + seen_timestamp, + ), } } @@ -702,30 +716,50 @@ impl SyncManager { } } - fn rpc_block_received( + fn rpc_block_or_blob_received( &mut self, request_id: RequestId, peer_id: PeerId, - beacon_block: Option>>, + block_or_blob: BlockOrBlob, seen_timestamp: Duration, ) { match request_id { - RequestId::SingleBlock { id } => self.block_lookups.single_block_lookup_response( - id, - peer_id, - beacon_block.map(|block| block.into()), - seen_timestamp, - &mut self.network, - ), - RequestId::ParentLookup { id } => self.block_lookups.parent_lookup_response( - id, - peer_id, - beacon_block.map(|block| block.into()), - seen_timestamp, - &mut self.network, - ), + RequestId::SingleBlock { id } => { + // TODO(diva) adjust when dealing with by root requests. This code is here to + // satisfy dead code analysis + match block_or_blob { + BlockOrBlob::Block(maybe_block) => { + self.block_lookups.single_block_lookup_response( + id, + peer_id, + maybe_block.map(BlockWrapper::Block), + seen_timestamp, + &mut self.network, + ) + } + BlockOrBlob::Sidecar(_) => unimplemented!("Mimatch between BlockWrapper and what the network receives needs to be handled first."), + } + } + RequestId::ParentLookup { id } => { + // TODO(diva) adjust when dealing with by root requests. This code is here to + // satisfy dead code analysis + match block_or_blob { + BlockOrBlob::Block(maybe_block) => self.block_lookups.parent_lookup_response( + id, + peer_id, + maybe_block.map(BlockWrapper::Block), + seen_timestamp, + &mut self.network, + ), + BlockOrBlob::Sidecar(_) => unimplemented!("Mimatch between BlockWrapper and what the network receives needs to be handled first."), + } + } RequestId::BackFillBlocks { id } => { - let is_stream_terminator = beacon_block.is_none(); + let maybe_block = match block_or_blob { + BlockOrBlob::Block(maybe_block) => maybe_block, + BlockOrBlob::Sidecar(_) => todo!("I think this is unreachable"), + }; + let is_stream_terminator = maybe_block.is_none(); if let Some(batch_id) = self .network .backfill_sync_only_blocks_response(id, is_stream_terminator) @@ -735,7 +769,7 @@ impl SyncManager { batch_id, &peer_id, id, - beacon_block.map(|block| block.into()), + maybe_block.map(|block| block.into()), ) { Ok(ProcessResult::SyncCompleted) => self.update_sync_state(), Ok(ProcessResult::Successful) => {} @@ -748,7 +782,11 @@ impl SyncManager { } } RequestId::RangeBlocks { id } => { - let is_stream_terminator = beacon_block.is_none(); + let maybe_block = match block_or_blob { + BlockOrBlob::Block(maybe_block) => maybe_block, + BlockOrBlob::Sidecar(_) => todo!("I think this should be unreachable, since this is a range only-blocks request, and the network should not accept this chunk at all. Needs better handling"), + }; + let is_stream_terminator = maybe_block.is_none(); if let Some((chain_id, batch_id)) = self .network .range_sync_block_response(id, is_stream_terminator) @@ -759,28 +797,28 @@ impl SyncManager { chain_id, batch_id, id, - beacon_block.map(|block| block.into()), + maybe_block.map(|block| block.into()), ); self.update_sync_state(); } } RequestId::BackFillBlobs { id } => { - self.blobs_backfill_response(id, peer_id, beacon_block.into()) + self.backfill_block_and_blobs_response(id, peer_id, block_or_blob) } RequestId::RangeBlobs { id } => { - self.blobs_range_response(id, peer_id, beacon_block.into()) + self.range_block_and_blobs_response(id, peer_id, block_or_blob) } } } /// Handles receiving a response for a range sync request that should have both blocks and /// blobs. - fn blobs_range_response( + fn range_block_and_blobs_response( &mut self, id: Id, peer_id: PeerId, - block_or_blob: BlockOrBlobs, + block_or_blob: BlockOrBlob, ) { if let Some((chain_id, resp)) = self .network @@ -822,11 +860,11 @@ impl SyncManager { /// Handles receiving a response for a Backfill sync request that should have both blocks and /// blobs. - fn blobs_backfill_response( + fn backfill_block_and_blobs_response( &mut self, id: Id, peer_id: PeerId, - block_or_blob: BlockOrBlobs, + block_or_blob: BlockOrBlob, ) { if let Some(resp) = self .network @@ -871,32 +909,6 @@ impl SyncManager { } } } - - fn rpc_blobs_received( - &mut self, - request_id: RequestId, - _peer_id: PeerId, - _maybe_blob: Option::EthSpec>>>, - _seen_timestamp: Duration, - ) { - match request_id { - RequestId::SingleBlock { .. } | RequestId::ParentLookup { .. } => { - unreachable!("There is no such thing as a singular 'by root' glob request that is not accompanied by the block") - } - RequestId::BackFillBlocks { .. } => { - unreachable!("An only blocks request does not receive sidecars") - } - RequestId::BackFillBlobs { .. } => { - unimplemented!("Adjust backfill sync"); - } - RequestId::RangeBlocks { .. } => { - unreachable!("Only-blocks range requests don't receive sidecars") - } - RequestId::RangeBlobs { id: _ } => { - unimplemented!("Adjust range"); - } - } - } } impl From>> for BlockProcessResult { diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 10f7f329557..974d8dbd8c8 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -75,20 +75,20 @@ pub struct SyncNetworkContext { } /// Small enumeration to make dealing with block and blob requests easier. -pub enum BlockOrBlobs { +pub enum BlockOrBlob { Block(Option>>), - Blobs(Option>>), + Sidecar(Option>>), } -impl From>>> for BlockOrBlobs { +impl From>>> for BlockOrBlob { fn from(block: Option>>) -> Self { - BlockOrBlobs::Block(block) + BlockOrBlob::Block(block) } } -impl From>>> for BlockOrBlobs { +impl From>>> for BlockOrBlob { fn from(blob: Option>>) -> Self { - BlockOrBlobs::Blobs(blob) + BlockOrBlob::Sidecar(blob) } } @@ -311,15 +311,15 @@ impl SyncNetworkContext { pub fn range_sync_block_and_blob_response( &mut self, request_id: Id, - block_or_blob: BlockOrBlobs, + block_or_blob: BlockOrBlob, ) -> Option<(ChainId, BlocksAndBlobsByRangeResponse)> { match self.range_blocks_and_blobs_requests.entry(request_id) { Entry::Occupied(mut entry) => { let req = entry.get_mut(); let info = &mut req.block_blob_info; match block_or_blob { - BlockOrBlobs::Block(maybe_block) => info.add_block_response(maybe_block), - BlockOrBlobs::Blobs(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar), + BlockOrBlob::Block(maybe_block) => info.add_block_response(maybe_block), + BlockOrBlob::Sidecar(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar), } if info.is_finished() { // If the request is finished, dequeue everything @@ -402,14 +402,14 @@ impl SyncNetworkContext { pub fn backfill_sync_block_and_blob_response( &mut self, request_id: Id, - block_or_blob: BlockOrBlobs, + block_or_blob: BlockOrBlob, ) -> Option> { match self.backfill_blocks_and_blobs_requests.entry(request_id) { Entry::Occupied(mut entry) => { let (_, info) = entry.get_mut(); match block_or_blob { - BlockOrBlobs::Block(maybe_block) => info.add_block_response(maybe_block), - BlockOrBlobs::Blobs(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar), + BlockOrBlob::Block(maybe_block) => info.add_block_response(maybe_block), + BlockOrBlob::Sidecar(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar), } if info.is_finished() { // If the request is finished, dequeue everything From 1301c62436d261ea646a86125584d227a618254c Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Sat, 18 Mar 2023 00:29:25 +1100 Subject: [PATCH 26/96] Validator blob signing for the unblinded flow (#4096) * Implement validator blob signing (full block and full blob) * Fix compilation error and remove redundant slot check * Fix clippy error --- common/eth2/src/types.rs | 20 +- consensus/types/src/chain_spec.rs | 14 +- consensus/types/src/config_and_preset.rs | 2 +- consensus/types/src/signed_blob.rs | 4 + validator_client/src/block_service.rs | 241 +++++++++++++---------- validator_client/src/signing_method.rs | 6 + validator_client/src/validator_store.rs | 45 ++++- 7 files changed, 218 insertions(+), 114 deletions(-) diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index db64d74c2ad..bc27ddb4743 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1326,7 +1326,7 @@ impl> Into> pub type BlockContentsTuple = ( SignedBeaconBlock, - Option, ::MaxBlobsPerBlock>>, + Option>, ); /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. @@ -1374,9 +1374,25 @@ impl> From> From> + for SignedBlockContents +{ + fn from(block_contents_tuple: BlockContentsTuple) -> Self { + match block_contents_tuple { + (signed_block, None) => SignedBlockContents::Block(signed_block), + (signed_block, Some(signed_blob_sidecars)) => { + SignedBlockContents::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { + signed_block, + signed_blob_sidecars, + }) + } + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec")] pub struct SignedBeaconBlockAndBlobSidecars> { pub signed_block: SignedBeaconBlock, - pub signed_blob_sidecars: VariableList, ::MaxBlobsPerBlock>, + pub signed_blob_sidecars: SignedBlobSidecarList, } diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 1f947c9e7b2..c107f790c50 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -14,7 +14,7 @@ pub enum Domain { BlsToExecutionChange, BeaconProposer, BeaconAttester, - BlobsSideCar, + BlobSidecar, Randao, Deposit, VoluntaryExit, @@ -100,7 +100,7 @@ pub struct ChainSpec { */ pub(crate) domain_beacon_proposer: u32, pub(crate) domain_beacon_attester: u32, - pub(crate) domain_blobs_sidecar: u32, + pub(crate) domain_blob_sidecar: u32, pub(crate) domain_randao: u32, pub(crate) domain_deposit: u32, pub(crate) domain_voluntary_exit: u32, @@ -366,7 +366,7 @@ impl ChainSpec { match domain { Domain::BeaconProposer => self.domain_beacon_proposer, Domain::BeaconAttester => self.domain_beacon_attester, - Domain::BlobsSideCar => self.domain_blobs_sidecar, + Domain::BlobSidecar => self.domain_blob_sidecar, Domain::Randao => self.domain_randao, Domain::Deposit => self.domain_deposit, Domain::VoluntaryExit => self.domain_voluntary_exit, @@ -574,7 +574,7 @@ impl ChainSpec { domain_voluntary_exit: 4, domain_selection_proof: 5, domain_aggregate_and_proof: 6, - domain_blobs_sidecar: 10, // 0x0a000000 + domain_blob_sidecar: 11, // 0x0B000000 /* * Fork choice @@ -809,7 +809,7 @@ impl ChainSpec { domain_voluntary_exit: 4, domain_selection_proof: 5, domain_aggregate_and_proof: 6, - domain_blobs_sidecar: 10, + domain_blob_sidecar: 11, /* * Fork choice @@ -1285,7 +1285,7 @@ mod tests { test_domain(Domain::BeaconProposer, spec.domain_beacon_proposer, &spec); test_domain(Domain::BeaconAttester, spec.domain_beacon_attester, &spec); - test_domain(Domain::BlobsSideCar, spec.domain_blobs_sidecar, &spec); + test_domain(Domain::BlobSidecar, spec.domain_blob_sidecar, &spec); test_domain(Domain::Randao, spec.domain_randao, &spec); test_domain(Domain::Deposit, spec.domain_deposit, &spec); test_domain(Domain::VoluntaryExit, spec.domain_voluntary_exit, &spec); @@ -1311,7 +1311,7 @@ mod tests { &spec, ); - test_domain(Domain::BlobsSideCar, spec.domain_blobs_sidecar, &spec); + test_domain(Domain::BlobSidecar, spec.domain_blob_sidecar, &spec); } fn apply_bit_mask(domain_bytes: [u8; 4], spec: &ChainSpec) -> u32 { diff --git a/consensus/types/src/config_and_preset.rs b/consensus/types/src/config_and_preset.rs index ac93818b9c3..957376c3d6a 100644 --- a/consensus/types/src/config_and_preset.rs +++ b/consensus/types/src/config_and_preset.rs @@ -78,7 +78,7 @@ pub fn get_extra_fields(spec: &ChainSpec) -> HashMap { "bls_withdrawal_prefix".to_uppercase() => u8_hex(spec.bls_withdrawal_prefix_byte), "domain_beacon_proposer".to_uppercase() => u32_hex(spec.domain_beacon_proposer), "domain_beacon_attester".to_uppercase() => u32_hex(spec.domain_beacon_attester), - "domain_blobs_sidecar".to_uppercase() => u32_hex(spec.domain_blobs_sidecar), + "domain_blob_sidecar".to_uppercase() => u32_hex(spec.domain_blob_sidecar), "domain_randao".to_uppercase()=> u32_hex(spec.domain_randao), "domain_deposit".to_uppercase()=> u32_hex(spec.domain_deposit), "domain_voluntary_exit".to_uppercase() => u32_hex(spec.domain_voluntary_exit), diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index 4121b8b7f29..f9ae4812478 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -2,6 +2,7 @@ use crate::{test_utils::TestRandom, BlobSidecar, EthSpec, Signature}; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; +use ssz_types::VariableList; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; @@ -25,3 +26,6 @@ pub struct SignedBlobSidecar { pub message: BlobSidecar, pub signature: Signature, } + +pub type SignedBlobSidecarList = + VariableList, ::MaxBlobsPerBlock>; diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 0eb9a07c394..5fa32d3f425 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -6,9 +6,11 @@ use crate::{ OfflineOnFailure, }; use crate::{http_metrics::metrics, validator_store::ValidatorStore}; +use bls::SignatureBytes; use environment::RuntimeContext; -use eth2::types::SignedBlockContents; -use slog::{crit, debug, error, info, trace, warn}; +use eth2::types::{BlockContents, SignedBlockContents}; +use eth2::BeaconNodeHttpClient; +use slog::{crit, debug, error, info, trace, warn, Logger}; use slot_clock::SlotClock; use std::ops::Deref; use std::sync::Arc; @@ -16,8 +18,8 @@ use std::time::Duration; use tokio::sync::mpsc; use tokio::time::sleep; use types::{ - AbstractExecPayload, BeaconBlock, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, - PublicKeyBytes, Slot, + AbstractExecPayload, BlindedPayload, BlockType, EthSpec, FullPayload, Graffiti, PublicKeyBytes, + Slot, }; #[derive(Debug)] @@ -342,80 +344,46 @@ impl BlockService { "slot" => slot.as_u64(), ); // Request block from first responsive beacon node. - let block = self + let block_contents = self .beacon_nodes .first_success( RequireSynced::No, OfflineOnFailure::Yes, - |beacon_node| async move { - let block: BeaconBlock = match Payload::block_type() { - BlockType::Full => { - let _get_timer = metrics::start_timer_vec( - &metrics::BLOCK_SERVICE_TIMES, - &[metrics::BEACON_BLOCK_HTTP_GET], - ); - beacon_node - .get_validator_blocks::( - slot, - randao_reveal_ref, - graffiti.as_ref(), - ) - .await - .map_err(|e| { - BlockError::Recoverable(format!( - "Error from beacon node when producing block: {:?}", - e - )) - })? - .data - .into() - } - BlockType::Blinded => { - let _get_timer = metrics::start_timer_vec( - &metrics::BLOCK_SERVICE_TIMES, - &[metrics::BLINDED_BEACON_BLOCK_HTTP_GET], - ); - beacon_node - .get_validator_blinded_blocks::( - slot, - randao_reveal_ref, - graffiti.as_ref(), - ) - .await - .map_err(|e| { - BlockError::Recoverable(format!( - "Error from beacon node when producing block: {:?}", - e - )) - })? - .data - } - }; - - info!( + move |beacon_node| { + Self::get_validator_block( + beacon_node, + slot, + randao_reveal_ref, + graffiti, + proposer_index, log, - "Received unsigned block"; - "slot" => slot.as_u64(), - ); - if proposer_index != Some(block.proposer_index()) { - return Err(BlockError::Recoverable( - "Proposer index does not match block proposer. Beacon chain re-orged" - .to_string(), - )); - } - - Ok::<_, BlockError>(block) + ) }, ) .await?; + let (block, maybe_blob_sidecars) = block_contents.deconstruct(); let signing_timer = metrics::start_timer(&metrics::BLOCK_SIGNING_TIMES); - let signed_block_contents: SignedBlockContents = self_ref + + let signed_block = self_ref .validator_store .sign_block::(*validator_pubkey_ref, block, current_slot) .await - .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))? - .into(); + .map_err(|e| BlockError::Recoverable(format!("Unable to sign block: {:?}", e)))?; + + let maybe_signed_blobs = match maybe_blob_sidecars { + Some(blob_sidecars) => Some( + self_ref + .validator_store + .sign_blobs(*validator_pubkey_ref, blob_sidecars) + .await + .map_err(|e| { + BlockError::Recoverable(format!("Unable to sign blob: {:?}", e)) + })?, + ), + None => None, + }; + let signing_time_ms = Duration::from_secs_f64(signing_timer.map_or(0.0, |t| t.stop_and_record())).as_millis(); @@ -426,46 +394,19 @@ impl BlockService { "signing_time_ms" => signing_time_ms, ); + let signed_block_contents = SignedBlockContents::from((signed_block, maybe_signed_blobs)); + // Publish block with first available beacon node. self.beacon_nodes .first_success( RequireSynced::No, OfflineOnFailure::Yes, |beacon_node| async { - match Payload::block_type() { - BlockType::Full => { - let _post_timer = metrics::start_timer_vec( - &metrics::BLOCK_SERVICE_TIMES, - &[metrics::BEACON_BLOCK_HTTP_POST], - ); - beacon_node - .post_beacon_blocks(&signed_block_contents) - .await - .map_err(|e| { - BlockError::Irrecoverable(format!( - "Error from beacon node when publishing block: {:?}", - e - )) - })? - } - BlockType::Blinded => { - let _post_timer = metrics::start_timer_vec( - &metrics::BLOCK_SERVICE_TIMES, - &[metrics::BLINDED_BEACON_BLOCK_HTTP_POST], - ); - beacon_node - // TODO: need to be adjusted for blobs - .post_beacon_blinded_blocks(signed_block_contents.signed_block()) - .await - .map_err(|e| { - BlockError::Irrecoverable(format!( - "Error from beacon node when publishing block: {:?}", - e - )) - })? - } - } - Ok::<_, BlockError>(()) + Self::publish_signed_block_contents::( + &signed_block_contents, + beacon_node, + ) + .await }, ) .await?; @@ -482,4 +423,106 @@ impl BlockService { Ok(()) } + + async fn publish_signed_block_contents>( + signed_block_contents: &SignedBlockContents, + beacon_node: &BeaconNodeHttpClient, + ) -> Result<(), BlockError> { + match Payload::block_type() { + BlockType::Full => { + let _post_timer = metrics::start_timer_vec( + &metrics::BLOCK_SERVICE_TIMES, + &[metrics::BEACON_BLOCK_HTTP_POST], + ); + beacon_node + .post_beacon_blocks(signed_block_contents) + .await + .map_err(|e| { + BlockError::Irrecoverable(format!( + "Error from beacon node when publishing block: {:?}", + e + )) + })? + } + BlockType::Blinded => { + let _post_timer = metrics::start_timer_vec( + &metrics::BLOCK_SERVICE_TIMES, + &[metrics::BLINDED_BEACON_BLOCK_HTTP_POST], + ); + todo!("need to be adjusted for blobs"); + // beacon_node + // .post_beacon_blinded_blocks(signed_block_contents.signed_block()) + // .await + // .map_err(|e| { + // BlockError::Irrecoverable(format!( + // "Error from beacon node when publishing block: {:?}", + // e + // )) + // })? + } + } + Ok::<_, BlockError>(()) + } + + async fn get_validator_block>( + beacon_node: &BeaconNodeHttpClient, + slot: Slot, + randao_reveal_ref: &SignatureBytes, + graffiti: Option, + proposer_index: Option, + log: &Logger, + ) -> Result, BlockError> { + let block_contents: BlockContents = match Payload::block_type() { + BlockType::Full => { + let _get_timer = metrics::start_timer_vec( + &metrics::BLOCK_SERVICE_TIMES, + &[metrics::BEACON_BLOCK_HTTP_GET], + ); + beacon_node + .get_validator_blocks::(slot, randao_reveal_ref, graffiti.as_ref()) + .await + .map_err(|e| { + BlockError::Recoverable(format!( + "Error from beacon node when producing block: {:?}", + e + )) + })? + .data + } + BlockType::Blinded => { + let _get_timer = metrics::start_timer_vec( + &metrics::BLOCK_SERVICE_TIMES, + &[metrics::BLINDED_BEACON_BLOCK_HTTP_GET], + ); + todo!("implement blinded flow for blobs"); + // beacon_node + // .get_validator_blinded_blocks::( + // slot, + // randao_reveal_ref, + // graffiti.as_ref(), + // ) + // .await + // .map_err(|e| { + // BlockError::Recoverable(format!( + // "Error from beacon node when producing block: {:?}", + // e + // )) + // })? + // .data + } + }; + + info!( + log, + "Received unsigned block"; + "slot" => slot.as_u64(), + ); + if proposer_index != Some(block_contents.block().proposer_index()) { + return Err(BlockError::Recoverable( + "Proposer index does not match block proposer. Beacon chain re-orged".to_string(), + )); + } + + Ok::<_, BlockError>(block_contents) + } } diff --git a/validator_client/src/signing_method.rs b/validator_client/src/signing_method.rs index ae9df080965..e428bffcff7 100644 --- a/validator_client/src/signing_method.rs +++ b/validator_client/src/signing_method.rs @@ -37,6 +37,7 @@ pub enum Error { pub enum SignableMessage<'a, T: EthSpec, Payload: AbstractExecPayload = FullPayload> { RandaoReveal(Epoch), BeaconBlock(&'a BeaconBlock), + BlobSidecar(&'a BlobSidecar), AttestationData(&'a AttestationData), SignedAggregateAndProof(&'a AggregateAndProof), SelectionProof(Slot), @@ -58,6 +59,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> SignableMessage<'a, T, Pay match self { SignableMessage::RandaoReveal(epoch) => epoch.signing_root(domain), SignableMessage::BeaconBlock(b) => b.signing_root(domain), + SignableMessage::BlobSidecar(b) => b.signing_root(domain), SignableMessage::AttestationData(a) => a.signing_root(domain), SignableMessage::SignedAggregateAndProof(a) => a.signing_root(domain), SignableMessage::SelectionProof(slot) => slot.signing_root(domain), @@ -180,6 +182,10 @@ impl SigningMethod { Web3SignerObject::RandaoReveal { epoch } } SignableMessage::BeaconBlock(block) => Web3SignerObject::beacon_block(block)?, + SignableMessage::BlobSidecar(_) => { + // https://github.com/ConsenSys/web3signer/issues/726 + unimplemented!("Web3Signer blob signing not implemented.") + } SignableMessage::AttestationData(a) => Web3SignerObject::Attestation(a), SignableMessage::SignedAggregateAndProof(a) => { Web3SignerObject::AggregateAndProof(a) diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 36a0d057342..294689e3c1c 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -6,6 +6,7 @@ use crate::{ Config, }; use account_utils::{validator_definitions::ValidatorDefinition, ZeroizeString}; +use eth2::types::VariableList; use parking_lot::{Mutex, RwLock}; use slashing_protection::{ interchange::Interchange, InterchangeError, NotSafe, Safe, SlashingDatabase, @@ -19,11 +20,12 @@ use std::sync::Arc; use task_executor::TaskExecutor; use types::{ attestation::Error as AttestationError, graffiti::GraffitiString, AbstractExecPayload, Address, - AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof, - Domain, Epoch, EthSpec, Fork, Graffiti, Hash256, Keypair, PublicKeyBytes, SelectionProof, - Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedRoot, - SignedValidatorRegistrationData, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, - SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, + AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, BlobSidecarList, ChainSpec, + ContributionAndProof, Domain, Epoch, EthSpec, Fork, Graffiti, Hash256, Keypair, PublicKeyBytes, + SelectionProof, Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecar, + SignedBlobSidecarList, SignedContributionAndProof, SignedRoot, SignedValidatorRegistrationData, + Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, SyncCommitteeMessage, + SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, }; use validator_dir::ValidatorDir; @@ -531,6 +533,39 @@ impl ValidatorStore { } } + pub async fn sign_blobs( + &self, + validator_pubkey: PublicKeyBytes, + blob_sidecars: BlobSidecarList, + ) -> Result, Error> { + let mut signed_blob_sidecars = Vec::new(); + + for blob_sidecar in blob_sidecars.into_iter() { + let slot = blob_sidecar.slot; + let signing_epoch = slot.epoch(E::slots_per_epoch()); + let signing_context = self.signing_context(Domain::BlobSidecar, signing_epoch); + let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; + + let signature = signing_method + .get_signature::>( + SignableMessage::BlobSidecar(&blob_sidecar), + signing_context, + &self.spec, + &self.task_executor, + ) + .await?; + + metrics::inc_counter_vec(&metrics::SIGNED_BLOBS_TOTAL, &[metrics::SUCCESS]); + + signed_blob_sidecars.push(SignedBlobSidecar { + message: blob_sidecar, + signature, + }); + } + + Ok(VariableList::from(signed_blob_sidecars)) + } + pub async fn sign_attestation( &self, validator_pubkey: PublicKeyBytes, From b40dceaae9a9ba3d7ba6504f56ad42f954868a0a Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 22 Mar 2023 01:56:32 +1100 Subject: [PATCH 27/96] Update get blobs endpoint to return a list of BlobSidecars (#4109) * Update get blobs endpoint to return BlobSidecarList * Update code comment * Update blob retrieval to return BlobSidecarList without Arc * Remove usage of BlobSidecarList type alias to avoid code conflicts * Add clippy allow exception --- beacon_node/beacon_chain/src/beacon_chain.rs | 17 +++++ beacon_node/http_api/src/block_id.rs | 21 +++--- beacon_node/http_api/src/lib.rs | 76 ++++++++++---------- common/eth2/src/lib.rs | 18 +++-- 4 files changed, 77 insertions(+), 55 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index dc7541b6385..016bda13ab3 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1057,6 +1057,23 @@ impl BeaconChain { .map(Some) } + // FIXME(jimmy): temporary method added to unblock API work. This method will be replaced by + // the `get_blobs` method below once the new blob sidecar structure (`BlobSidecarList`) is + // implemented in that method. + #[allow(clippy::type_complexity)] // FIXME: this will be fixed by the `BlobSidecarList` alias in Sean's PR + pub fn get_blob_sidecar_list( + &self, + _block_root: &Hash256, + _data_availability_boundary: Epoch, + ) -> Result< + Option< + VariableList>, ::MaxBlobsPerBlock>, + >, + Error, + > { + unimplemented!("update to use the updated `get_blobs` method instead once this PR is merged: https://github.com/sigp/lighthouse/pull/4104") + } + /// Returns the blobs at the given root, if any. /// /// Returns `Ok(None)` if the blobs and associated block are not found. diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index b484f4079aa..9183437f990 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -1,10 +1,10 @@ use crate::{state_id::checkpoint_slot_and_execution_optimistic, ExecutionOptimistic}; use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes, WhenSlotSkipped}; -use eth2::types::BlockId as CoreBlockId; +use eth2::types::{BlockId as CoreBlockId, VariableList}; use std::fmt; use std::str::FromStr; use std::sync::Arc; -use types::{BlobsSidecar, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, Slot}; +use types::{BlobSidecar, EthSpec, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, Slot}; /// Wraps `eth2::types::BlockId` and provides a simple way to obtain a block or root for a given /// `BlockId`. @@ -212,19 +212,22 @@ impl BlockId { } } - /// Return the `BlobsSidecar` identified by `self`. - pub async fn blobs_sidecar( + /// Return the `BlobSidecarList` identified by `self`. + pub async fn blob_sidecar_list( &self, chain: &BeaconChain, - ) -> Result>, warp::Rejection> { + ) -> Result< + VariableList>, ::MaxBlobsPerBlock>, + warp::Rejection, + > { let root = self.root(chain)?.0; let Some(data_availability_boundary) = chain.data_availability_boundary() else { - return Err(warp_utils::reject::custom_not_found("Eip4844 fork disabled".into())); + return Err(warp_utils::reject::custom_not_found("Deneb fork disabled".into())); }; - match chain.get_blobs(&root, data_availability_boundary) { - Ok(Some(blob)) => Ok(Arc::new(blob)), + match chain.get_blob_sidecar_list(&root, data_availability_boundary) { + Ok(Some(blobs)) => Ok(blobs), Ok(None) => Err(warp_utils::reject::custom_not_found(format!( - "Blob with block root {} is not in the store", + "No blobs with block root {} found in the store", root ))), Err(e) => Err(warp_utils::reject::beacon_chain_error(e)), diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 6b0518a23c0..797e8f72b45 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1293,6 +1293,45 @@ pub fn serve( }, ); + /* + * beacon/blobs + */ + + // GET beacon/blobs/{block_id} + let get_blobs = eth_v1 + .and(warp::path("beacon")) + .and(warp::path("blobs")) + .and(block_id_or_err) + .and(warp::path::end()) + .and(chain_filter.clone()) + .and(warp::header::optional::("accept")) + .and_then( + |block_id: BlockId, + chain: Arc>, + accept_header: Option| { + async move { + let blob_sidecar_list = block_id.blob_sidecar_list(&chain).await?; + + match accept_header { + Some(api_types::Accept::Ssz) => Response::builder() + .status(200) + .header("Content-Type", "application/octet-stream") + .body(blob_sidecar_list.as_ssz_bytes().into()) + .map_err(|e| { + warp_utils::reject::custom_server_error(format!( + "failed to create response: {}", + e + )) + }), + _ => Ok(warp::reply::json(&api_types::GenericResponse::from( + blob_sidecar_list, + )) + .into_response()), + } + } + }, + ); + /* * beacon/pool */ @@ -3498,41 +3537,6 @@ pub fn serve( ) }); - // GET lighthouse/beacon/blobs_sidecars/{block_id} - let get_lighthouse_blobs_sidecars = warp::path("lighthouse") - .and(warp::path("beacon")) - .and(warp::path("blobs_sidecars")) - .and(block_id_or_err) - .and(warp::path::end()) - .and(chain_filter.clone()) - .and(warp::header::optional::("accept")) - .and_then( - |block_id: BlockId, - chain: Arc>, - accept_header: Option| { - async move { - let blobs_sidecar = block_id.blobs_sidecar(&chain).await?; - - match accept_header { - Some(api_types::Accept::Ssz) => Response::builder() - .status(200) - .header("Content-Type", "application/octet-stream") - .body(blobs_sidecar.as_ssz_bytes().into()) - .map_err(|e| { - warp_utils::reject::custom_server_error(format!( - "failed to create response: {}", - e - )) - }), - _ => Ok(warp::reply::json(&api_types::GenericResponse::from( - blobs_sidecar, - )) - .into_response()), - } - } - }, - ); - let get_events = eth_v1 .and(warp::path("events")) .and(warp::path::end()) @@ -3627,6 +3631,7 @@ pub fn serve( .uor(get_beacon_block_attestations) .uor(get_beacon_blinded_block) .uor(get_beacon_block_root) + .uor(get_blobs) .uor(get_beacon_pool_attestations) .uor(get_beacon_pool_attester_slashings) .uor(get_beacon_pool_proposer_slashings) @@ -3672,7 +3677,6 @@ pub fn serve( .uor(get_lighthouse_attestation_performance) .uor(get_lighthouse_block_packing_efficiency) .uor(get_lighthouse_merge_readiness) - .uor(get_lighthouse_blobs_sidecars.boxed()) .uor(get_events) .recover(warp_utils::reject::handle_rejection), ) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 2a27d31da9b..a57c2ca3d71 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -658,15 +658,13 @@ impl BeaconNodeHttpClient { Ok(path) } - /// Path for `lighthouse/beacon/blobs_sidecars/{block_id}` - pub fn get_blobs_sidecar_path(&self, block_id: BlockId) -> Result { - let mut path = self.server.full.clone(); - + /// Path for `v1/beacon/blobs/{block_id}` + pub fn get_blobs_path(&self, block_id: BlockId) -> Result { + let mut path = self.eth_path(V1)?; path.path_segments_mut() .map_err(|()| Error::InvalidUrl(self.server.clone()))? - .push("lighthouse") .push("beacon") - .push("blobs_sidecars") + .push("blobs") .push(&block_id.to_string()); Ok(path) } @@ -698,14 +696,14 @@ impl BeaconNodeHttpClient { Ok(Some(response.json().await?)) } - /// `GET lighthouse/beacon/blobs_sidecars/{block_id}` + /// `GET v1/beacon/blobs/{block_id}` /// /// Returns `Ok(None)` on a 404 error. - pub async fn get_blobs_sidecar( + pub async fn get_blobs( &self, block_id: BlockId, - ) -> Result>>, Error> { - let path = self.get_blobs_sidecar_path(block_id)?; + ) -> Result>>, Error> { + let path = self.get_blobs_path(block_id)?; let response = match self.get_response(path, |b| b).await.optional()? { Some(res) => res, None => return Ok(None), From d1e653cfdbac04eddb69396f2a52322140ab5e04 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Tue, 21 Mar 2023 14:33:06 -0500 Subject: [PATCH 28/96] Update Blob Storage Structure (#4104) * Initial Changes to Blob Storage * Add Arc to SignedBlobSidecar Definition --- beacon_node/beacon_chain/src/beacon_chain.rs | 65 ++++----------- .../beacon_chain/src/early_attester_cache.rs | 4 +- beacon_node/http_api/src/block_id.rs | 13 ++- .../beacon_processor/worker/rpc_methods.rs | 80 +++++++------------ beacon_node/store/src/hot_cold_store.rs | 20 ++--- beacon_node/store/src/lib.rs | 3 +- consensus/tree_hash/src/impls.rs | 19 +++++ consensus/types/src/blob_sidecar.rs | 3 +- consensus/types/src/signed_blob.rs | 3 +- 9 files changed, 86 insertions(+), 124 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 016bda13ab3..00be8a25c97 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -959,35 +959,20 @@ impl BeaconChain { Ok(self.get_block(block_root).await?.map(Arc::new)) } - pub async fn get_block_and_blobs_checking_early_attester_cache( + pub async fn get_blobs_checking_early_attester_cache( &self, block_root: &Hash256, - ) -> Result>, Error> { + ) -> Result>, Error> { // If there is no data availability boundary, the Eip4844 fork is disabled. if let Some(finalized_data_availability_boundary) = self.finalized_data_availability_boundary() { - // Only use the attester cache if we can find both the block and blob - if let (Some(block), Some(blobs)) = ( - self.early_attester_cache.get_block(*block_root), - self.early_attester_cache.get_blobs(*block_root), - ) { - Ok(Some(SignedBeaconBlockAndBlobsSidecar { - beacon_block: block, - blobs_sidecar: blobs, - })) - // Attempt to get the block and blobs from the database - } else if let Some(block) = self.get_block(block_root).await?.map(Arc::new) { - let blobs = self - .get_blobs(block_root, finalized_data_availability_boundary)? - .map(Arc::new); - Ok(blobs.map(|blobs| SignedBeaconBlockAndBlobsSidecar { - beacon_block: block, - blobs_sidecar: blobs, - })) - } else { - Ok(None) - } + self.early_attester_cache + .get_blobs(*block_root) + .map_or_else( + || self.get_blobs(block_root, finalized_data_availability_boundary), + |blobs| Ok(Some(blobs)), + ) } else { Ok(None) } @@ -1057,23 +1042,6 @@ impl BeaconChain { .map(Some) } - // FIXME(jimmy): temporary method added to unblock API work. This method will be replaced by - // the `get_blobs` method below once the new blob sidecar structure (`BlobSidecarList`) is - // implemented in that method. - #[allow(clippy::type_complexity)] // FIXME: this will be fixed by the `BlobSidecarList` alias in Sean's PR - pub fn get_blob_sidecar_list( - &self, - _block_root: &Hash256, - _data_availability_boundary: Epoch, - ) -> Result< - Option< - VariableList>, ::MaxBlobsPerBlock>, - >, - Error, - > { - unimplemented!("update to use the updated `get_blobs` method instead once this PR is merged: https://github.com/sigp/lighthouse/pull/4104") - } - /// Returns the blobs at the given root, if any. /// /// Returns `Ok(None)` if the blobs and associated block are not found. @@ -1091,9 +1059,9 @@ impl BeaconChain { &self, block_root: &Hash256, data_availability_boundary: Epoch, - ) -> Result>, Error> { + ) -> Result>, Error> { match self.store.get_blobs(block_root)? { - Some(blobs) => Ok(Some(blobs)), + Some(blob_sidecar_list) => Ok(Some(blob_sidecar_list)), None => { // Check for the corresponding block to understand whether we *should* have blobs. self.get_blinded_block(block_root)? @@ -1106,7 +1074,8 @@ impl BeaconChain { Err(_) => return Err(Error::NoKzgCommitmentsFieldOnBlock), }; if expected_kzg_commitments.is_empty() { - Ok(BlobsSidecar::empty_from_parts(*block_root, block.slot())) + // TODO (mark): verify this + Ok(BlobSidecarList::empty()) } else if data_availability_boundary <= block.epoch() { // We should have blobs for all blocks younger than the boundary. Err(Error::BlobsUnavailable) @@ -3052,7 +3021,7 @@ impl BeaconChain { // margin, or younger (of higher epoch number). if block_epoch >= import_boundary { if let Some(blobs) = blobs { - if !blobs.blobs.is_empty() { + if !blobs.is_empty() { //FIXME(sean) using this for debugging for now info!( self.log, "Writing blobs to store"; @@ -4814,7 +4783,7 @@ impl BeaconChain { ) .map_err(BlockProductionError::KzgError)?; - let blob_sidecars = VariableList::from( + let blob_sidecars = BlobSidecarList::from( blobs .into_iter() .enumerate() @@ -4827,7 +4796,7 @@ impl BeaconChain { .get(blob_index) .expect("KZG proof should exist for blob"); - Ok(BlobSidecar { + Ok(Arc::new(BlobSidecar { block_root: beacon_block_root, index: blob_index as u64, slot, @@ -4836,9 +4805,9 @@ impl BeaconChain { blob, kzg_commitment: *kzg_commitment, kzg_proof: *kzg_proof, - }) + })) }) - .collect::>, BlockProductionError>>()?, + .collect::, BlockProductionError>>()?, ); self.blob_cache.put(beacon_block_root, blob_sidecars); diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index dd4109da9b4..5fe14c7e252 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -21,7 +21,7 @@ pub struct CacheItem { * Values used to make the block available. */ block: Arc>, - blobs: Option>>, + blobs: Option>, proto_block: ProtoBlock, } @@ -160,7 +160,7 @@ impl EarlyAttesterCache { } /// Returns the blobs, if `block_root` matches the cached item. - pub fn get_blobs(&self, block_root: Hash256) -> Option>> { + pub fn get_blobs(&self, block_root: Hash256) -> Option> { self.item .read() .as_ref() diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index 9183437f990..ef7affeb7f4 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -1,10 +1,10 @@ use crate::{state_id::checkpoint_slot_and_execution_optimistic, ExecutionOptimistic}; use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes, WhenSlotSkipped}; -use eth2::types::{BlockId as CoreBlockId, VariableList}; +use eth2::types::BlockId as CoreBlockId; use std::fmt; use std::str::FromStr; use std::sync::Arc; -use types::{BlobSidecar, EthSpec, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, Slot}; +use types::{BlobSidecarList, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, Slot}; /// Wraps `eth2::types::BlockId` and provides a simple way to obtain a block or root for a given /// `BlockId`. @@ -216,16 +216,13 @@ impl BlockId { pub async fn blob_sidecar_list( &self, chain: &BeaconChain, - ) -> Result< - VariableList>, ::MaxBlobsPerBlock>, - warp::Rejection, - > { + ) -> Result, warp::Rejection> { let root = self.root(chain)?.0; let Some(data_availability_boundary) = chain.data_availability_boundary() else { return Err(warp_utils::reject::custom_not_found("Deneb fork disabled".into())); }; - match chain.get_blob_sidecar_list(&root, data_availability_boundary) { - Ok(Some(blobs)) => Ok(blobs), + match chain.get_blobs(&root, data_availability_boundary) { + Ok(Some(blob_sidecar_list)) => Ok(blob_sidecar_list), Ok(None) => Err(warp_utils::reject::custom_not_found(format!( "No blobs with block root {} found in the store", root diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 78b9de303fc..565b1ce8867 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -12,6 +12,7 @@ use lighthouse_network::rpc::*; use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo}; use slog::{debug, error, warn}; use slot_clock::SlotClock; +use std::collections::{hash_map::Entry, HashMap}; use std::sync::Arc; use task_executor::TaskExecutor; use types::blob_sidecar::BlobIdentifier; @@ -225,42 +226,34 @@ impl Worker { executor.spawn( async move { let requested_blobs = request.blob_ids.len(); - let mut send_block_count = 0; + let mut send_blob_count = 0; let mut send_response = true; + + let mut blob_list_results = HashMap::new(); for BlobIdentifier{ block_root: root, index } in request.blob_ids.into_iter() { - match self - .chain - .get_block_and_blobs_checking_early_attester_cache(&root) - .await - { - Ok(Some(block_and_blobs)) => { - // - // TODO: HORRIBLE NSFW CODE AHEAD - // - let types::SignedBeaconBlockAndBlobsSidecar {beacon_block, blobs_sidecar} = block_and_blobs; - let types::BlobsSidecar{ beacon_block_root, beacon_block_slot, blobs: blob_bundle, kzg_aggregated_proof }: types::BlobsSidecar<_> = blobs_sidecar.as_ref().clone(); - // TODO: this should be unreachable after this is addressed seriously, - // so for now let's be ok with a panic in the expect. - let block = beacon_block.message_eip4844().expect("We fucked up the block blob stuff"); - // Intentionally not accessing the list directly - for (known_index, blob) in blob_bundle.into_iter().enumerate() { - if (known_index as u64) == index { - let blob_sidecar = types::BlobSidecar{ - block_root: beacon_block_root, - index, - slot: beacon_block_slot, - block_parent_root: block.parent_root, - proposer_index: block.proposer_index, - blob, - kzg_commitment: block.body.blob_kzg_commitments[known_index], // TODO: needs to be stored in a more logical way so that this won't panic. - kzg_proof: kzg_aggregated_proof // TODO: yeah - }; + let blob_list_result = match blob_list_results.entry(root) { + Entry::Vacant(entry) => { + entry.insert(self + .chain + .get_blobs_checking_early_attester_cache(&root) + .await) + } + Entry::Occupied(entry) => { + entry.into_mut() + } + }; + + match blob_list_result.as_ref() { + Ok(Some(blobs_sidecar_list)) => { + for blob_sidecar in blobs_sidecar_list.iter() { + if blob_sidecar.index == index { self.send_response( peer_id, - Response::BlobsByRoot(Some(Arc::new(blob_sidecar))), + Response::BlobsByRoot(Some(blob_sidecar.clone())), request_id, ); - send_block_count += 1; + send_blob_count += 1; + break; } } } @@ -355,7 +348,7 @@ impl Worker { "Received BlobsByRoot Request"; "peer" => %peer_id, "requested" => requested_blobs, - "returned" => send_block_count + "returned" => send_blob_count ); // send stream termination @@ -837,31 +830,12 @@ impl Worker { for root in block_roots { match self.chain.get_blobs(&root, data_availability_boundary) { - Ok(Some(blobs)) => { - // TODO: more GROSS code ahead. Reader beware - let types::BlobsSidecar { - beacon_block_root, - beacon_block_slot, - blobs: blob_bundle, - kzg_aggregated_proof: _, - }: types::BlobsSidecar<_> = blobs; - - for (blob_index, blob) in blob_bundle.into_iter().enumerate() { - let blob_sidecar = types::BlobSidecar { - block_root: beacon_block_root, - index: blob_index as u64, - slot: beacon_block_slot, - block_parent_root: Hash256::zero(), - proposer_index: 0, - blob, - kzg_commitment: types::KzgCommitment::default(), - kzg_proof: types::KzgProof::default(), - }; - + Ok(Some(blob_sidecar_list)) => { + for blob_sidecar in blob_sidecar_list.iter() { blobs_sent += 1; self.send_network_message(NetworkMessage::SendResponse { peer_id, - response: Response::BlobsByRange(Some(Arc::new(blob_sidecar))), + response: Response::BlobsByRange(Some(blob_sidecar.clone())), id: request_id, }); } diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 60e2f775959..2c80c2c1ac7 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -66,7 +66,7 @@ pub struct HotColdDB, Cold: ItemStore> { /// The hot database also contains all blocks. pub hot_db: Hot, /// LRU cache of deserialized blobs. Updated whenever a blob is loaded. - blob_cache: Mutex>>, + blob_cache: Mutex>>, /// LRU cache of deserialized blocks. Updated whenever a block is loaded. block_cache: Mutex>>, /// Chain spec. @@ -568,7 +568,7 @@ impl, Cold: ItemStore> HotColdDB blobs_db.key_delete(DBColumn::BeaconBlob.into(), block_root.as_bytes()) } - pub fn put_blobs(&self, block_root: &Hash256, blobs: BlobsSidecar) -> Result<(), Error> { + pub fn put_blobs(&self, block_root: &Hash256, blobs: BlobSidecarList) -> Result<(), Error> { let blobs_db = self.blobs_db.as_ref().unwrap_or(&self.cold_db); blobs_db.put_bytes( DBColumn::BeaconBlob.into(), @@ -582,7 +582,7 @@ impl, Cold: ItemStore> HotColdDB pub fn blobs_as_kv_store_ops( &self, key: &Hash256, - blobs: &BlobsSidecar, + blobs: BlobSidecarList, ops: &mut Vec, ) { let db_key = get_key_for_col(DBColumn::BeaconBlob.into(), key.as_bytes()); @@ -817,7 +817,7 @@ impl, Cold: ItemStore> HotColdDB } StoreOp::PutBlobs(block_root, blobs) => { - self.blobs_as_kv_store_ops(&block_root, &blobs, &mut key_value_batch); + self.blobs_as_kv_store_ops(&block_root, blobs, &mut key_value_batch); } StoreOp::PutStateSummary(state_root, summary) => { @@ -885,8 +885,8 @@ impl, Cold: ItemStore> HotColdDB StoreOp::PutBlobs(_, _) => true, StoreOp::DeleteBlobs(block_root) => { match self.get_blobs(block_root) { - Ok(Some(blobs_sidecar)) => { - blobs_to_delete.push(blobs_sidecar); + Ok(Some(blobs_sidecar_list)) => { + blobs_to_delete.push((*block_root, blobs_sidecar_list)); } Err(e) => { error!( @@ -926,7 +926,7 @@ impl, Cold: ItemStore> HotColdDB let reverse_op = match op { StoreOp::PutBlobs(block_root, _) => StoreOp::DeleteBlobs(*block_root), StoreOp::DeleteBlobs(_) => match blobs_to_delete.pop() { - Some(blobs) => StoreOp::PutBlobs(blobs.beacon_block_root, Arc::new(blobs)), + Some((block_root, blobs)) => StoreOp::PutBlobs(block_root, blobs), None => return Err(HotColdDBError::Rollback.into()), }, _ => return Err(HotColdDBError::Rollback.into()), @@ -972,7 +972,7 @@ impl, Cold: ItemStore> HotColdDB for op in blob_cache_ops { match op { StoreOp::PutBlobs(block_root, blobs) => { - guard_blob.put(block_root, (*blobs).clone()); + guard_blob.put(block_root, blobs); } StoreOp::DeleteBlobs(block_root) => { @@ -1320,12 +1320,12 @@ impl, Cold: ItemStore> HotColdDB } /// Fetch a blobs sidecar from the store. - pub fn get_blobs(&self, block_root: &Hash256) -> Result>, Error> { + pub fn get_blobs(&self, block_root: &Hash256) -> Result>, Error> { let blobs_db = self.blobs_db.as_ref().unwrap_or(&self.cold_db); match blobs_db.get_bytes(DBColumn::BeaconBlob.into(), block_root.as_bytes())? { Some(ref blobs_bytes) => { - let blobs = BlobsSidecar::from_ssz_bytes(blobs_bytes)?; + let blobs = BlobSidecarList::from_ssz_bytes(blobs_bytes)?; // FIXME(sean) I was attempting to use a blob cache here but was getting deadlocks, // may want to attempt to use one again self.blob_cache.lock().put(*block_root, blobs.clone()); diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 3056c292923..29fded5fa6d 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -159,7 +159,8 @@ pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'stati pub enum StoreOp<'a, E: EthSpec> { PutBlock(Hash256, Arc>), PutState(Hash256, &'a BeaconState), - PutBlobs(Hash256, Arc>), + // TODO (mark): space can be optimized here by de-duplicating data + PutBlobs(Hash256, BlobSidecarList), PutOrphanedBlobsKey(Hash256), PutStateSummary(Hash256, HotStateSummary), PutStateTemporaryFlag(Hash256), diff --git a/consensus/tree_hash/src/impls.rs b/consensus/tree_hash/src/impls.rs index 899356f8331..134be402194 100644 --- a/consensus/tree_hash/src/impls.rs +++ b/consensus/tree_hash/src/impls.rs @@ -1,5 +1,6 @@ use super::*; use ethereum_types::{H160, H256, U128, U256}; +use std::sync::Arc; fn int_to_hash256(int: u64) -> Hash256 { let mut bytes = [0; HASHSIZE]; @@ -186,6 +187,24 @@ impl TreeHash for H256 { } } +impl TreeHash for Arc { + fn tree_hash_type() -> TreeHashType { + T::tree_hash_type() + } + + fn tree_hash_packed_encoding(&self) -> PackedEncoding { + self.as_ref().tree_hash_packed_encoding() + } + + fn tree_hash_packing_factor() -> usize { + T::tree_hash_packing_factor() + } + + fn tree_hash_root(&self) -> Hash256 { + self.as_ref().tree_hash_root() + } +} + #[cfg(test)] mod test { use super::*; diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 169c570d291..29eaadc5842 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -6,6 +6,7 @@ use serde_derive::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; +use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; @@ -47,7 +48,7 @@ pub struct BlobSidecar { pub kzg_proof: KzgProof, } -pub type BlobSidecarList = VariableList, ::MaxBlobsPerBlock>; +pub type BlobSidecarList = VariableList>, ::MaxBlobsPerBlock>; impl SignedRoot for BlobSidecar {} diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index f9ae4812478..4eb28794ed5 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -3,6 +3,7 @@ use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; +use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; @@ -23,7 +24,7 @@ use tree_hash_derive::TreeHash; #[arbitrary(bound = "T: EthSpec")] #[derivative(Hash(bound = "T: EthSpec"))] pub struct SignedBlobSidecar { - pub message: BlobSidecar, + pub message: Arc>, pub signature: Signature, } From b276af98b75bfd4f2f92e1da022864ee49dcbdc3 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Sat, 25 Mar 2023 03:00:41 +0530 Subject: [PATCH 29/96] Rework block processing (#4092) * introduce availability pending block * add intoavailableblock trait * small fixes * add 'gossip blob cache' and start to clean up processing and transition types * shard memory blob cache * Initial commit * Fix after rebase * Add gossip verification conditions * cache cleanup * general chaos * extended chaos * cargo fmt * more progress * more progress * tons of changes, just tryna compile * everything, everywhere, all at once * Reprocess an ExecutedBlock on unavailable blobs * Add sus gossip verification for blobs * Merge stuff * Remove reprocessing cache stuff * lint * Add a wrapper to allow construction of only valid `AvailableBlock`s * rename blob arc list to blob list * merge cleanuo * Revert "merge cleanuo" This reverts commit 5e98326878c77528d0c4668c5a4db4a4b0fbaeaa. * Revert "Revert "merge cleanuo"" This reverts commit 3a4009443a5812b3028abe855079307436dc5419. * fix rpc methods * move beacon block and blob to eth2/types * rename gossip blob cache to data availability checker * lots of changes * fix some compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * cargo fmt * use a common data structure for block import types * fix availability check on proposal import * refactor the blob cache and split the block wrapper into two types * add type conversion for signed block and block wrapper * fix beacon chain tests and do some renaming, add some comments * Partial processing (#4) * move beacon block and blob to eth2/types * rename gossip blob cache to data availability checker * lots of changes * fix some compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * fix compilation issues * cargo fmt * use a common data structure for block import types * fix availability check on proposal import * refactor the blob cache and split the block wrapper into two types * add type conversion for signed block and block wrapper * fix beacon chain tests and do some renaming, add some comments * cargo update (#6) --------- Co-authored-by: realbigsean Co-authored-by: realbigsean --- Cargo.lock | 1004 +++++++++-------- beacon_node/beacon_chain/src/beacon_chain.rs | 396 ++++--- .../beacon_chain/src/blob_verification.rs | 729 ++++++------ .../beacon_chain/src/block_verification.rs | 190 +++- beacon_node/beacon_chain/src/builder.rs | 16 +- .../src/data_availability_checker.rs | 516 +++++++++ .../beacon_chain/src/early_attester_cache.rs | 4 +- beacon_node/beacon_chain/src/lib.rs | 10 +- beacon_node/beacon_chain/src/test_utils.rs | 6 +- .../tests/attestation_production.rs | 27 +- .../beacon_chain/tests/block_verification.rs | 3 +- .../tests/payload_invalidation.rs | 4 + beacon_node/beacon_chain/tests/tests.rs | 27 +- .../src/engine_api/json_structures.rs | 5 +- beacon_node/execution_layer/src/lib.rs | 5 +- beacon_node/http_api/src/block_id.rs | 5 +- .../http_api/src/build_block_contents.rs | 6 +- beacon_node/http_api/src/publish_blocks.rs | 71 +- .../lighthouse_network/src/rpc/protocol.rs | 13 +- beacon_node/network/Cargo.toml | 2 +- .../network/src/beacon_processor/mod.rs | 10 +- .../work_reprocessing_queue.rs | 43 +- .../beacon_processor/worker/gossip_methods.rs | 88 +- .../beacon_processor/worker/rpc_methods.rs | 5 +- .../beacon_processor/worker/sync_methods.rs | 33 +- beacon_node/network/src/router.rs | 1 - .../network/src/sync/block_lookups/mod.rs | 3 +- .../src/sync/block_lookups/parent_lookup.rs | 3 +- .../sync/block_lookups/single_block_lookup.rs | 3 +- beacon_node/network/src/sync/manager.rs | 14 +- beacon_node/store/src/hot_cold_store.rs | 3 +- .../store/src/impls/execution_payload.rs | 4 +- beacon_node/store/src/lib.rs | 1 + common/eth2/src/types.rs | 29 + .../state_processing/src/consensus_context.rs | 2 +- .../src/beacon_block_and_blob_sidecars.rs | 37 - consensus/types/src/beacon_block_body.rs | 4 +- consensus/types/src/blob_sidecar.rs | 11 +- consensus/types/src/blobs_sidecar.rs | 62 - consensus/types/src/lib.rs | 8 - consensus/types/src/signed_beacon_block.rs | 30 - consensus/types/src/signed_blob.rs | 41 +- consensus/types/src/signed_block_and_blobs.rs | 35 - lcli/src/parse_ssz.rs | 2 +- testing/ef_tests/src/type_name.rs | 6 +- testing/ef_tests/tests/tests.rs | 7 - 46 files changed, 2102 insertions(+), 1422 deletions(-) create mode 100644 beacon_node/beacon_chain/src/data_availability_checker.rs delete mode 100644 consensus/types/src/beacon_block_and_blob_sidecars.rs delete mode 100644 consensus/types/src/blobs_sidecar.rs delete mode 100644 consensus/types/src/signed_block_and_blobs.rs diff --git a/Cargo.lock b/Cargo.lock index 2303b483922..37bdb4294b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "aead" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c192eb8f11fc081b0fe4259ba5af04217d4e0faddd02417310a927911abd7c8" +dependencies = [ + "crypto-common", + "generic-array", +] + [[package]] name = "aes" version = "0.6.0" @@ -113,17 +123,14 @@ dependencies = [ ] [[package]] -name = "aes-gcm" -version = "0.8.0" +name = "aes" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" +checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" dependencies = [ - "aead 0.3.2", - "aes 0.6.0", - "cipher 0.2.5", - "ctr 0.6.0", - "ghash 0.3.1", - "subtle", + "cfg-if", + "cipher 0.4.4", + "cpufeatures", ] [[package]] @@ -140,6 +147,20 @@ dependencies = [ "subtle", ] +[[package]] +name = "aes-gcm" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e1366e0c69c9f927b1fa5ce2c7bf9eafc8f9268c0b9800729e8b267612447c" +dependencies = [ + "aead 0.5.1", + "aes 0.8.2", + "cipher 0.4.4", + "ctr 0.9.2", + "ghash 0.5.0", + "subtle", +] + [[package]] name = "aes-soft" version = "0.6.4" @@ -205,9 +226,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.69" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "arbitrary" @@ -225,9 +246,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "arrayref" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" [[package]] name = "arrayvec" @@ -248,14 +269,14 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.17", + "time 0.3.20", ] [[package]] name = "asn1-rs" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf6690c370453db30743b373a60ba498fc0d6d83b11f4abfd87a84a075db5dd4" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" dependencies = [ "asn1-rs-derive 0.4.0", "asn1-rs-impl", @@ -264,7 +285,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -275,7 +296,7 @@ checksum = "db8b7511298d5b7784b40b092d9e9dcd3a627a5707e4b5e507931ab0d44eeebf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "synstructure", ] @@ -287,7 +308,7 @@ checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "synstructure", ] @@ -299,7 +320,7 @@ checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -310,64 +331,64 @@ checksum = "e22d1f4b888c298a027c99dc9048015fac177587de20fc30232a057dfbe24a21" [[package]] name = "async-io" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c374dda1ed3e7d8f0d9ba58715f924862c63eae6849c92d3a18e7fbde9e2794" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" dependencies = [ "async-lock", "autocfg 1.1.0", + "cfg-if", "concurrent-queue", "futures-lite", - "libc", "log", "parking", "polling", + "rustix 0.37.3", "slab", "socket2", "waker-fn", - "windows-sys 0.42.0", ] [[package]] name = "async-lock" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" dependencies = [ "event-listener", - "futures-lite", ] [[package]] name = "async-stream" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" dependencies = [ "async-stream-impl", "futures-core", + "pin-project-lite 0.2.9", ] [[package]] name = "async-stream-impl" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] @@ -431,7 +452,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -537,14 +558,14 @@ checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" [[package]] name = "base64ct" -version = "1.5.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "beacon-api-client" version = "0.1.0" -source = "git+https://github.com/ralexstokes/beacon-api-client#53690a711e33614d59d4d44fb09762b4699e2a4e" +source = "git+https://github.com/ralexstokes/beacon-api-client#30679e9e25d61731cde54e14cd8a3688a39d8e5b" dependencies = [ "ethereum-consensus", "http", @@ -734,9 +755,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] @@ -980,9 +1001,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", "js-sys", @@ -1011,11 +1032,21 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clang-sys" -version = "1.4.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" +checksum = "77ed9a53e5d4d9c573ae844bfac6872b159cb1d1585a83b29e7a64b7eef7332a" dependencies = [ "glob", "libc", @@ -1088,7 +1119,7 @@ dependencies = [ "state_processing", "store", "task_executor", - "time 0.3.17", + "time 0.3.20", "timer", "tokio", "types", @@ -1125,7 +1156,7 @@ name = "compare_fields_derive" version = "0.2.0" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1149,9 +1180,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cec318a675afcb6a1ea1d4340e2d377e56e47c266f28043ceccbf4412ddfdd3b" +checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" [[package]] name = "convert_case" @@ -1193,12 +1224,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpuid-bool" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" - [[package]] name = "crc" version = "3.0.1" @@ -1261,9 +1286,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1271,9 +1296,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -1282,22 +1307,22 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg 1.1.0", "cfg-if", "crossbeam-utils", - "memoffset 0.7.1", + "memoffset 0.8.0", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", ] @@ -1327,6 +1352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1340,16 +1366,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "crypto-mac" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "crypto-mac" version = "0.11.1" @@ -1362,9 +1378,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af91f40b7355f82b0a891f50e70399475945bb0b0da4f1700ce60761c9d3e359" +checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" dependencies = [ "csv-core", "itoa", @@ -1383,20 +1399,20 @@ dependencies = [ [[package]] name = "ctr" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" dependencies = [ - "cipher 0.2.5", + "cipher 0.3.0", ] [[package]] name = "ctr" -version = "0.8.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher 0.3.0", + "cipher 0.4.4", ] [[package]] @@ -1424,9 +1440,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.0.0-rc.0" +version = "4.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da00a7a9a4eb92a0a0f8e75660926d48f0d0f3c537e455c457bcdaa1e16b1ac" +checksum = "8d4ba9852b42210c7538b75484f9daa0655e9a3ac04f693747bb0f02cf3cfe16" dependencies = [ "cfg-if", "fiat-crypto", @@ -1438,9 +1454,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.90" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90d59d9acd2a682b4e40605a242f6670eaa58c5957471cbf85e8aa6a0b97a5e8" +checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" dependencies = [ "cc", "cxxbridge-flags", @@ -1450,9 +1466,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.90" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfa40bda659dd5c864e65f4c9a2b0aff19bea56b017b9b77c73d3766a453a38" +checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" dependencies = [ "cc", "codespan-reporting", @@ -1460,24 +1476,24 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 2.0.9", ] [[package]] name = "cxxbridge-flags" -version = "1.0.90" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457ce6757c5c70dc6ecdbda6925b958aae7f959bda7d8fb9bde889e34a09dc03" +checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" [[package]] name = "cxxbridge-macro" -version = "1.0.90" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebf883b7aacd7b2aeb2a7b338648ee19f57c140d4ee8e52c68979c6b2f7f2263" +checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] @@ -1492,12 +1508,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ - "darling_core 0.14.3", - "darling_macro 0.14.3", + "darling_core 0.14.4", + "darling_macro 0.14.4", ] [[package]] @@ -1511,21 +1527,21 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn", + "syn 1.0.109", ] [[package]] name = "darling_core" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.10.0", - "syn", + "syn 1.0.109", ] [[package]] @@ -1536,18 +1552,18 @@ checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core 0.13.4", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "darling_macro" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ - "darling_core 0.14.3", + "darling_core 0.14.4", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1593,7 +1609,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" dependencies = [ "data-encoding", - "syn", + "syn 1.0.109", ] [[package]] @@ -1671,11 +1687,11 @@ dependencies = [ [[package]] name = "der-parser" -version = "8.1.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d4bc9b0db0a0df9ae64634ac5bdefb7afcb534e182275ca0beadbe486701c1" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" dependencies = [ - "asn1-rs 0.5.1", + "asn1-rs 0.5.2", "displaydoc", "nom 7.1.3", "num-bigint", @@ -1691,7 +1707,7 @@ checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1699,10 +1715,10 @@ name = "derive_arbitrary" version = "1.2.2" source = "git+https://github.com/michaelsproul/arbitrary?rev=a572fd8743012a4f1ada5ee5968b1b3619c427ba#a572fd8743012a4f1ada5ee5968b1b3619c427ba" dependencies = [ - "darling 0.14.3", + "darling 0.14.4", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1720,10 +1736,10 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ - "darling 0.14.3", + "darling 0.14.4", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1733,7 +1749,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" dependencies = [ "derive_builder_core", - "syn", + "syn 1.0.109", ] [[package]] @@ -1746,7 +1762,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.0", - "syn", + "syn 1.0.109", ] [[package]] @@ -1764,7 +1780,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ - "block-buffer 0.10.3", + "block-buffer 0.10.4", "crypto-common", "subtle", ] @@ -1861,14 +1877,14 @@ checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "dtoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00704156a7de8df8da0911424e30c2049957b0a714542a44e05fe693dd85313" +checksum = "65d09067bfacaa79114679b279d7f5885b53295b1e2cfb4e79c8e4bd3d633169" [[package]] name = "ecdsa" @@ -2021,7 +2037,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2080,6 +2096,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "errno" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.45.0", +] + [[package]] name = "errno-dragonfly" version = "0.1.2" @@ -2285,7 +2312,7 @@ dependencies = [ "eth2_ssz", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2402,7 +2429,7 @@ dependencies = [ "hex", "integer-sqrt", "multiaddr 0.14.0", - "multihash", + "multihash 0.16.3", "rand 0.8.5", "serde", "serde_json", @@ -2634,18 +2661,18 @@ checksum = "ec54ac60a7f2ee9a97cad9946f9bf629a3bc6a7ae59e68983dc9318f5a54b81a" [[package]] name = "fiat-crypto" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a214f5bb88731d436478f3ae1f8a277b62124089ba9fb67f4f93fb100ef73c90" +checksum = "93ace6ec7cc19c8ed33a32eaa9ea692d7faea05006b5356b9e2b668ec4bc3955" [[package]] name = "field-offset" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e1c54951450cbd39f3dbcf1005ac413b49487dabf18a720ad2383eccfeffb92" +checksum = "a3cf3a800ff6e860c863ca6d4b16fd999db8b752819c1606884047b73e468535" dependencies = [ - "memoffset 0.6.5", - "rustc_version 0.3.3", + "memoffset 0.8.0", + "rustc_version 0.4.0", ] [[package]] @@ -2767,9 +2794,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" +checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" dependencies = [ "futures-channel", "futures-core", @@ -2782,9 +2809,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" dependencies = [ "futures-core", "futures-sink", @@ -2792,15 +2819,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" [[package]] name = "futures-executor" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" +checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" dependencies = [ "futures-core", "futures-task", @@ -2810,9 +2837,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" [[package]] name = "futures-lite" @@ -2831,13 +2858,13 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" +checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2853,15 +2880,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" [[package]] name = "futures-task" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" [[package]] name = "futures-timer" @@ -2871,9 +2898,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" dependencies = [ "futures-channel", "futures-core", @@ -2955,29 +2982,29 @@ dependencies = [ [[package]] name = "ghash" -version = "0.3.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" +checksum = "1583cc1656d7839fd3732b80cf4f38850336cdb9b8ded1cd399ca62958de3c99" dependencies = [ "opaque-debug", - "polyval 0.4.5", + "polyval 0.5.3", ] [[package]] name = "ghash" -version = "0.4.4" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1583cc1656d7839fd3732b80cf4f38850336cdb9b8ded1cd399ca62958de3c99" +checksum = "d930750de5717d2dd0b8c0d42c076c0e884c81a73e6cab859bbd2339c71e3e40" dependencies = [ "opaque-debug", - "polyval 0.5.3", + "polyval 0.6.0", ] [[package]] name = "gimli" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "221996f774192f0f718773def8201c4ae31f02616a54ccfc2d358bb0e5cefdec" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" [[package]] name = "git-version" @@ -2998,7 +3025,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -3020,9 +3047,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -3152,6 +3179,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "hex" version = "0.4.3" @@ -3183,16 +3216,6 @@ dependencies = [ "digest 0.9.0", ] -[[package]] -name = "hmac" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" -dependencies = [ - "crypto-mac 0.10.1", - "digest 0.9.0", -] - [[package]] name = "hmac" version = "0.11.0" @@ -3236,9 +3259,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -3349,9 +3372,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.24" +version = "0.14.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" dependencies = [ "bytes", "futures-channel", @@ -3399,16 +3422,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows 0.46.0", ] [[package]] @@ -3495,7 +3518,7 @@ dependencies = [ "rtnetlink", "system-configuration", "tokio", - "windows", + "windows 0.34.0", ] [[package]] @@ -3564,7 +3587,7 @@ checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -3577,6 +3600,15 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -3628,10 +3660,11 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ + "hermit-abi 0.3.1", "libc", "windows-sys 0.45.0", ] @@ -3665,9 +3698,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "jemalloc-ctl" @@ -3726,11 +3759,11 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "8.2.0" +version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f4f04699947111ec1733e71778d763555737579e44b85844cae8e1940a1828" +checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64 0.13.1", + "base64 0.21.0", "pem", "ring", "serde", @@ -3866,15 +3899,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.139" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "libflate" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05605ab2bce11bcfc0e9c635ff29ef8b2ea83f29be257ee7d730cac3ee373093" +checksum = "97822bf791bd4d5b403713886a5fbe8bf49520fe78e323b0dc480ca1a03e50b0" dependencies = [ "adler32", "crc32fast", @@ -3883,9 +3916,9 @@ dependencies = [ [[package]] name = "libflate_lz77" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39a734c0493409afcd49deee13c006a04e3586b9761a03543c6272c9c51f2f5a" +checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" dependencies = [ "rle-decode-fast", ] @@ -3929,9 +3962,9 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.50.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0a0d2f693675f49ded13c5d510c48b78069e23cbd9108d7ccd59f6dc568819" +checksum = "9c7b0104790be871edcf97db9bd2356604984e623a08d825c3f27852290266b8" dependencies = [ "bytes", "futures", @@ -3977,7 +4010,7 @@ dependencies = [ "libsecp256k1", "log", "multiaddr 0.14.0", - "multihash", + "multihash 0.16.3", "multistream-select 0.11.0", "p256", "parking_lot 0.12.1", @@ -4011,7 +4044,7 @@ dependencies = [ "libsecp256k1", "log", "multiaddr 0.16.0", - "multihash", + "multihash 0.16.3", "multistream-select 0.12.1", "once_cell", "p256", @@ -4030,6 +4063,34 @@ dependencies = [ "zeroize", ] +[[package]] +name = "libp2p-core" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7f8b7d65c070a5a1b5f8f0510648189da08f787b8963f8e21219e0710733af" +dependencies = [ + "either", + "fnv", + "futures", + "futures-timer", + "instant", + "libp2p-identity", + "log", + "multiaddr 0.17.1", + "multihash 0.17.0", + "multistream-select 0.12.1", + "once_cell", + "parking_lot 0.12.1", + "pin-project", + "quick-protobuf", + "rand 0.8.5", + "rw-stream-sink", + "smallvec", + "thiserror", + "unsigned-varint 0.7.1", + "void", +] + [[package]] name = "libp2p-dns" version = "0.38.0" @@ -4095,6 +4156,24 @@ dependencies = [ "void", ] +[[package]] +name = "libp2p-identity" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a8ea433ae0cea7e3315354305237b9897afe45278b2118a7a57ca744e70fd27" +dependencies = [ + "bs58", + "ed25519-dalek", + "log", + "multiaddr 0.17.1", + "multihash 0.17.0", + "prost", + "quick-protobuf", + "rand 0.8.5", + "thiserror", + "zeroize", +] + [[package]] name = "libp2p-mdns" version = "0.42.0" @@ -4237,7 +4316,7 @@ checksum = "9d527d5827582abd44a6d80c07ff8b50b4ee238a8979e05998474179e79dc400" dependencies = [ "heck", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -4258,13 +4337,14 @@ dependencies = [ [[package]] name = "libp2p-tls" -version = "0.1.0-alpha" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7905ce0d040576634e8a3229a7587cc8beab83f79db6023800f1792895defa8" +checksum = "ff08d13d0dc66e5e9ba6279c1de417b84fa0d0adc3b03e5732928c180ec02781" dependencies = [ "futures", "futures-rustls", - "libp2p-core 0.38.0", + "libp2p-core 0.39.1", + "libp2p-identity", "rcgen 0.10.0", "ring", "rustls 0.20.8", @@ -4290,7 +4370,7 @@ dependencies = [ "libp2p-core 0.38.0", "libp2p-noise", "log", - "multihash", + "multihash 0.16.3", "prost", "prost-build", "prost-codec", @@ -4538,6 +4618,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +[[package]] +name = "linux-raw-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd550e73688e6d578f0ac2119e32b797a327631a42f9433e59d02e139c8df60d" + [[package]] name = "lmdb-rkv" version = "0.14.0" @@ -4722,9 +4808,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg 1.1.0", ] @@ -4761,7 +4847,7 @@ dependencies = [ "proc-macro2", "quote", "smallvec", - "syn", + "syn 1.0.109", ] [[package]] @@ -4795,9 +4881,9 @@ dependencies = [ [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" @@ -4872,7 +4958,7 @@ dependencies = [ "bs58", "byteorder", "data-encoding", - "multihash", + "multihash 0.16.3", "percent-encoding", "serde", "static_assertions", @@ -4890,7 +4976,26 @@ dependencies = [ "byteorder", "data-encoding", "multibase", - "multihash", + "multihash 0.16.3", + "percent-encoding", + "serde", + "static_assertions", + "unsigned-varint 0.7.1", + "url", +] + +[[package]] +name = "multiaddr" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b36f567c7099511fa8612bbbb52dda2419ce0bdbacf31714e3a5ffdb766d3bd" +dependencies = [ + "arrayref", + "byteorder", + "data-encoding", + "log", + "multibase", + "multihash 0.17.0", "percent-encoding", "serde", "static_assertions", @@ -4922,6 +5027,19 @@ dependencies = [ "unsigned-varint 0.7.1", ] +[[package]] +name = "multihash" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835d6ff01d610179fbce3de1694d007e500bf33a7f29689838941d6bf783ae40" +dependencies = [ + "core2", + "digest 0.10.6", + "multihash-derive", + "sha2 0.10.6", + "unsigned-varint 0.7.1", +] + [[package]] name = "multihash-derive" version = "0.8.1" @@ -4932,7 +5050,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "synstructure", ] @@ -5061,9 +5179,9 @@ dependencies = [ [[package]] name = "netlink-sys" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "260e21fbb6f3d253a14df90eb0000a6066780a15dd901a7519ce02d77a94985b" +checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411" dependencies = [ "bytes", "futures", @@ -5114,7 +5232,7 @@ dependencies = [ "task_executor", "tokio", "tokio-stream", - "tokio-util 0.6.10", + "tokio-util 0.7.7", "types", ] @@ -5314,7 +5432,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" dependencies = [ - "asn1-rs 0.5.1", + "asn1-rs 0.5.2", ] [[package]] @@ -5364,14 +5482,14 @@ dependencies = [ "bytes", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "openssl" -version = "0.10.45" +version = "0.10.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2" dependencies = [ "bitflags", "cfg-if", @@ -5390,7 +5508,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -5401,18 +5519,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.25.1+1.1.1t" +version = "111.25.2+1.1.1t" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ef9a9cc6ea7d9d5e7c4a913dc4b48d0e359eddf01af1dfec96ba7064b4aba10" +checksum = "320708a054ad9b3bf314688b5db87cf4d6683d64cfc835e2337924ae62bf4431" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.80" +version = "0.9.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" dependencies = [ "autocfg 1.1.0", "cc", @@ -5521,7 +5639,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -5533,7 +5651,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -5592,9 +5710,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" [[package]] name = "pbkdf2" @@ -5644,16 +5762,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" -[[package]] -name = "pest" -version = "2.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028accff104c4e513bad663bbcd2ad7cfd5304144404c31ed0a77ac103d00660" -dependencies = [ - "thiserror", - "ucd-trie", -] - [[package]] name = "petgraph" version = "0.6.3" @@ -5691,7 +5799,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -5770,16 +5878,18 @@ dependencies = [ [[package]] name = "polling" -version = "2.5.2" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22122d5ec4f9fe1b3916419b76be1e80bcb93f618d071d2edf841b137b2a2bd6" +checksum = "7e1f879b2998099c2d69ab9605d145d5b661195627eccc680002c4918a7fb6fa" dependencies = [ "autocfg 1.1.0", + "bitflags", "cfg-if", + "concurrent-queue", "libc", "log", - "wepoll-ffi", - "windows-sys 0.42.0", + "pin-project-lite 0.2.9", + "windows-sys 0.45.0", ] [[package]] @@ -5790,30 +5900,31 @@ checksum = "048aeb476be11a4b6ca432ca569e375810de9294ae78f4774e78ea98a9246ede" dependencies = [ "cpufeatures", "opaque-debug", - "universal-hash", + "universal-hash 0.4.1", ] [[package]] name = "polyval" -version = "0.4.5" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" +checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" dependencies = [ - "cpuid-bool", + "cfg-if", + "cpufeatures", "opaque-debug", - "universal-hash", + "universal-hash 0.4.1", ] [[package]] name = "polyval" -version = "0.5.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" +checksum = "7ef234e08c11dfcb2e56f79fd70f6f2eb7f025c0ce2333e82f4f0518ecad30c6" dependencies = [ "cfg-if", "cpufeatures", "opaque-debug", - "universal-hash", + "universal-hash 0.5.0", ] [[package]] @@ -5824,12 +5935,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.1.23" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" dependencies = [ "proc-macro2", - "syn", + "syn 1.0.109", ] [[package]] @@ -5878,7 +5989,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -5901,9 +6012,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" dependencies = [ "unicode-ident", ] @@ -5955,14 +6066,14 @@ checksum = "66a455fbcb954c1a7decf3c586e860fd7889cddf4b8e164be736dbac95a953cd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "prost" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698" +checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537" dependencies = [ "bytes", "prost-derive", @@ -5970,9 +6081,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f8ad728fb08fe212df3c05169e940fbb6d9d16a877ddde14644a983ba2012e" +checksum = "2c828f93f5ca4826f97fedcbd3f9a536c16b12cff3dbbb4a007f932bbad95b12" dependencies = [ "bytes", "heck", @@ -5985,7 +6096,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 1.0.109", "tempfile", "which", ] @@ -6005,24 +6116,23 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda8c0881ea9f722eb9629376db3d0b903b462477c1aafcb0566610ac28ac5d" +checksum = "4ea9b0f8cbe5e15a8a042d030bd96668db28ecb567ec37d691971ff5731d2b1b" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "prost-types" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788" +checksum = "379119666929a1afd7a043aa6cf96fa67a6dce9af60c88095a4686dbce4c9c88" dependencies = [ - "bytes", "prost", ] @@ -6070,6 +6180,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-protobuf" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +dependencies = [ + "byteorder", +] + [[package]] name = "quickcheck" version = "0.9.2" @@ -6090,7 +6209,7 @@ checksum = "608c156fd8e97febc07dc9c2e2c80bf74cfc6ef26893eae3daf8bc2bc94a4b7f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -6124,9 +6243,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -6246,9 +6365,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -6256,9 +6375,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -6274,7 +6393,7 @@ checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" dependencies = [ "pem", "ring", - "time 0.3.17", + "time 0.3.20", "x509-parser 0.13.2", "yasna", ] @@ -6287,7 +6406,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring", - "time 0.3.17", + "time 0.3.20", "yasna", ] @@ -6313,9 +6432,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "cce168fea28d3e05f158bda4576cf0c844d5045bc2cc3620fa0292ed5bb5814c" dependencies = [ "aho-corasick", "memchr", @@ -6333,15 +6452,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949" dependencies = [ "base64 0.21.0", "bytes", @@ -6441,7 +6560,7 @@ checksum = "e33d7b2abe0c340d8797fe2907d3f20d3b5ea5908683618bfe80df7f621f672a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -6510,9 +6629,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +checksum = "d4a36c42d1873f9a77c53bde094f9664d9891bc604a45b4798fd2c389ed12e5b" [[package]] name = "rustc-hash" @@ -6535,22 +6654,13 @@ dependencies = [ "semver 0.9.0", ] -[[package]] -name = "rustc_version" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" -dependencies = [ - "semver 0.11.0", -] - [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.16", + "semver 1.0.17", ] [[package]] @@ -6564,15 +6674,29 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.9" +version = "0.36.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" +dependencies = [ + "bitflags", + "errno 0.2.8", + "io-lifetimes", + "libc", + "linux-raw-sys 0.1.4", + "windows-sys 0.45.0", +] + +[[package]] +name = "rustix" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" +checksum = "62b24138615de35e32031d041a09032ef3487a616d901ca4db224e7d557efae2" dependencies = [ "bitflags", - "errno", + "errno 0.3.0", "io-lifetimes", "libc", - "linux-raw-sys", + "linux-raw-sys 0.3.0", "windows-sys 0.45.0", ] @@ -6612,9 +6736,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" [[package]] name = "rw-stream-sink" @@ -6629,9 +6753,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "safe_arith" @@ -6663,9 +6787,9 @@ dependencies = [ [[package]] name = "scale-info" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "001cf62ece89779fd16105b5f515ad0e5cedcd5440d3dd806bb067978e7c3608" +checksum = "61471dff9096de1d8b2319efed7162081e96793f5ebb147e50db10d50d648a4d" dependencies = [ "cfg-if", "derive_more", @@ -6675,14 +6799,14 @@ dependencies = [ [[package]] name = "scale-info-derive" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "303959cf613a6f6efd19ed4b4ad5bf79966a13352716299ad532cfb115f4205c" +checksum = "219580e803a66b3f05761fd06f1f879a872444e49ce23f73694d26e5a954c7e6" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -6696,9 +6820,9 @@ dependencies = [ [[package]] name = "scheduled-thread-pool" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" dependencies = [ "parking_lot 0.12.1", ] @@ -6717,9 +6841,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "scrypt" @@ -6826,23 +6950,14 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser 0.7.0", -] - -[[package]] -name = "semver" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" -dependencies = [ - "semver-parser 0.10.2", + "semver-parser", ] [[package]] name = "semver" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "semver-parser" @@ -6850,15 +6965,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -[[package]] -name = "semver-parser" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" -dependencies = [ - "pest", -] - [[package]] name = "send_wrapper" version = "0.6.0" @@ -6875,9 +6981,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" dependencies = [ "serde_derive", ] @@ -6904,20 +7010,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ "itoa", "ryu", @@ -6926,13 +7032,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.10" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a5ec9fa74a20ebbe5d9ac23dac1fc96ba0ecfe9f50f2843b52e537b10fbcb4e" +checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] @@ -6966,7 +7072,7 @@ dependencies = [ "darling 0.13.4", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -7105,7 +7211,7 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -7128,9 +7234,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg 1.1.0", ] @@ -7231,7 +7337,7 @@ dependencies = [ "serde", "serde_json", "slog", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -7276,7 +7382,7 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -7327,14 +7433,14 @@ checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" [[package]] name = "snow" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ba5f4d4ff12bdb6a169ed51b7c48c0e0ac4b0b4b31012b2571e97d78d3201d" +checksum = "5ccba027ba85743e09d15c03296797cad56395089b832b48b5a5217880f57733" dependencies = [ "aes-gcm 0.9.4", "blake2", "chacha20poly1305", - "curve25519-dalek 4.0.0-rc.0", + "curve25519-dalek 4.0.0-rc.1", "rand_core 0.6.4", "ring", "rustc_version 0.4.0", @@ -7344,9 +7450,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -7405,7 +7511,7 @@ source = "git+https://github.com/ralexstokes//ssz-rs?rev=adf1a0b14cef90b9536f28e dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -7509,7 +7615,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 1.0.109", ] [[package]] @@ -7557,7 +7663,7 @@ dependencies = [ "proc-macro2", "quote", "smallvec", - "syn", + "syn 1.0.109", ] [[package]] @@ -7571,7 +7677,7 @@ dependencies = [ "proc-macro2", "quote", "smallvec", - "syn", + "syn 1.0.109", ] [[package]] @@ -7585,9 +7691,20 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "0da4a3c17e109f700685ec577c0f85efd9b19bcf15c913985f14dc1ac01775aa" dependencies = [ "proc-macro2", "quote", @@ -7608,7 +7725,7 @@ checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "unicode-xid", ] @@ -7708,7 +7825,7 @@ dependencies = [ "cfg-if", "fastrand", "redox_syscall", - "rustix", + "rustix 0.36.11", "windows-sys 0.42.0", ] @@ -7745,7 +7862,7 @@ name = "test_random_derive" version = "0.2.0" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -7759,22 +7876,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] @@ -7809,9 +7926,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", "libc", @@ -7829,9 +7946,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -7938,7 +8055,7 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -7975,9 +8092,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite 0.2.9", @@ -8123,7 +8240,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -8191,7 +8308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebeb235c5847e2f82cfe0f07eb971d1e5f6804b18dac2ae16349cc604380f82f" dependencies = [ "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -8215,7 +8332,7 @@ version = "0.4.0" dependencies = [ "darling 0.13.4", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -8406,12 +8523,6 @@ dependencies = [ "tree_hash_derive", ] -[[package]] -name = "ucd-trie" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" - [[package]] name = "uint" version = "0.9.5" @@ -8442,15 +8553,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.10" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -8483,6 +8594,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "universal-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3160b73c9a19f7e2939a2fdad446c57c1bbbbf4d919d3213ff1267a580d8b5" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "unsigned-varint" version = "0.6.0" @@ -8668,12 +8789,11 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -8774,7 +8894,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -8808,7 +8928,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -9008,7 +9128,7 @@ dependencies = [ "sha2 0.10.6", "stun", "thiserror", - "time 0.3.17", + "time 0.3.20", "tokio", "turn", "url", @@ -9040,22 +9160,22 @@ dependencies = [ [[package]] name = "webrtc-dtls" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7021987ae0a2ed6c8cd33f68e98e49bb6e74ffe9543310267b48a1bbe3900e5f" +checksum = "942be5bd85f072c3128396f6e5a9bfb93ca8c1939ded735d177b7bcba9a13d05" dependencies = [ "aes 0.6.0", - "aes-gcm 0.8.0", + "aes-gcm 0.10.1", "async-trait", "bincode", "block-modes", "byteorder", "ccm", "curve25519-dalek 3.2.0", - "der-parser 8.1.0", + "der-parser 8.2.0", "elliptic-curve", "hkdf", - "hmac 0.10.1", + "hmac 0.12.1", "log", "oid-registry 0.6.1", "p256", @@ -9067,8 +9187,8 @@ dependencies = [ "rustls 0.19.1", "sec1", "serde", - "sha-1 0.9.8", - "sha2 0.9.9", + "sha1", + "sha2 0.10.6", "signature", "subtle", "thiserror", @@ -9194,15 +9314,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "wepoll-ffi" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" -dependencies = [ - "cc", -] - [[package]] name = "which" version = "4.4.0" @@ -9270,6 +9381,15 @@ dependencies = [ "windows_x86_64_msvc 0.34.0", ] +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-acl" version = "0.3.0" @@ -9289,12 +9409,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.1", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -9308,24 +9428,24 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.1", + "windows_x86_64_msvc 0.42.2", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_msvc" @@ -9335,9 +9455,9 @@ checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_i686_gnu" @@ -9347,9 +9467,9 @@ checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_msvc" @@ -9359,9 +9479,9 @@ checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_x86_64_gnu" @@ -9371,15 +9491,15 @@ checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_msvc" @@ -9389,9 +9509,9 @@ checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "winreg" @@ -9474,7 +9594,7 @@ dependencies = [ "ring", "rusticata-macros", "thiserror", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -9483,16 +9603,16 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0ecbeb7b67ce215e40e3cc7f2ff902f94a223acf44995934763467e7b1febc8" dependencies = [ - "asn1-rs 0.5.1", + "asn1-rs 0.5.2", "base64 0.13.1", "data-encoding", - "der-parser 8.1.0", + "der-parser 8.2.0", "lazy_static", "nom 7.1.3", "oid-registry 0.6.1", "rusticata-macros", "thiserror", - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -9539,7 +9659,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aed2e7a52e3744ab4d0c05c20aa065258e84c49fd4226f5191b2ed29712710b4" dependencies = [ - "time 0.3.17", + "time 0.3.20", ] [[package]] @@ -9559,7 +9679,7 @@ checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "synstructure", ] diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a23061fff7e..1d4f1d17e97 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -8,15 +8,19 @@ use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache} use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; use crate::blob_cache::BlobCache; -use crate::blob_verification::{AsBlock, AvailableBlock, BlockWrapper}; +use crate::blob_verification::{self, AsBlock, BlobError, BlockWrapper, GossipVerifiedBlob}; use crate::block_times_cache::BlockTimesCache; +use crate::block_verification::POS_PANDA_BANNER; use crate::block_verification::{ check_block_is_finalized_checkpoint_or_descendant, check_block_relevancy, get_block_root, - signature_verify_chain_segment, BlockError, ExecutionPendingBlock, GossipVerifiedBlock, - IntoExecutionPendingBlock, PayloadVerificationOutcome, POS_PANDA_BANNER, + signature_verify_chain_segment, AvailableExecutedBlock, BlockError, BlockImportData, + ExecutedBlock, ExecutionPendingBlock, GossipVerifiedBlock, IntoExecutionPendingBlock, }; pub use crate::canonical_head::{CanonicalHead, CanonicalHeadRwLock}; use crate::chain_config::ChainConfig; +use crate::data_availability_checker::{ + Availability, AvailabilityCheckError, AvailableBlock, DataAvailabilityChecker, +}; use crate::early_attester_cache::EarlyAttesterCache; use crate::errors::{BeaconChainError as Error, BlockProductionError}; use crate::eth1_chain::{Eth1Chain, Eth1ChainBackend}; @@ -109,8 +113,9 @@ use store::{ use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; use tree_hash::TreeHash; +use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; -use types::blobs_sidecar::KzgCommitments; +use types::blob_sidecar::{BlobIdentifier, BlobSidecarList, Blobs}; use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; use types::consts::merge::INTERVALS_PER_SLOT; use types::*; @@ -183,6 +188,36 @@ pub enum WhenSlotSkipped { Prev, } +#[derive(Debug, PartialEq)] +pub enum AvailabilityProcessingStatus { + PendingBlobs(Vec), + PendingBlock(Hash256), + Imported(Hash256), +} + +//TODO(sean) using this in tests for now +impl TryInto for AvailabilityProcessingStatus { + type Error = (); + + fn try_into(self) -> Result { + match self { + AvailabilityProcessingStatus::Imported(hash) => Ok(hash.into()), + _ => Err(()), + } + } +} + +impl TryInto for AvailabilityProcessingStatus { + type Error = (); + + fn try_into(self) -> Result { + match self { + AvailabilityProcessingStatus::Imported(hash) => Ok(hash), + _ => Err(()), + } + } +} + /// The result of a chain segment processing. pub enum ChainSegmentResult { /// Processing this chain segment finished successfully. @@ -433,8 +468,9 @@ pub struct BeaconChain { pub slasher: Option>>, /// Provides monitoring of a set of explicitly defined validators. pub validator_monitor: RwLock>, - pub blob_cache: BlobCache, - pub kzg: Option>, + pub proposal_blob_cache: BlobCache, + pub data_availability_checker: DataAvailabilityChecker, + pub kzg: Option>, } type BeaconBlockAndState = (BeaconBlock, BeaconState); @@ -991,19 +1027,9 @@ impl BeaconChain { &self, block_root: &Hash256, ) -> Result>, Error> { - // If there is no data availability boundary, the Eip4844 fork is disabled. - if let Some(finalized_data_availability_boundary) = - self.finalized_data_availability_boundary() - { - self.early_attester_cache - .get_blobs(*block_root) - .map_or_else( - || self.get_blobs(block_root, finalized_data_availability_boundary), - |blobs| Ok(Some(blobs)), - ) - } else { - Ok(None) - } + self.early_attester_cache + .get_blobs(*block_root) + .map_or_else(|| self.get_blobs(block_root), |blobs| Ok(Some(blobs))) } /// Returns the block at the given root, if any. @@ -1086,35 +1112,8 @@ impl BeaconChain { pub fn get_blobs( &self, block_root: &Hash256, - data_availability_boundary: Epoch, ) -> Result>, Error> { - match self.store.get_blobs(block_root)? { - Some(blob_sidecar_list) => Ok(Some(blob_sidecar_list)), - None => { - // Check for the corresponding block to understand whether we *should* have blobs. - self.get_blinded_block(block_root)? - .map(|block| { - // If there are no KZG commitments in the block, we know the sidecar should - // be empty. - let expected_kzg_commitments = - match block.message().body().blob_kzg_commitments() { - Ok(kzg_commitments) => kzg_commitments, - Err(_) => return Err(Error::NoKzgCommitmentsFieldOnBlock), - }; - if expected_kzg_commitments.is_empty() { - // TODO (mark): verify this - Ok(BlobSidecarList::empty()) - } else if data_availability_boundary <= block.epoch() { - // We should have blobs for all blocks younger than the boundary. - Err(Error::BlobsUnavailable) - } else { - // We shouldn't have blobs for blocks older than the boundary. - Err(Error::BlobsOlderThanDataAvailabilityBoundary(block.epoch())) - } - }) - .transpose() - } - } + Ok(self.store.get_blobs(block_root)?) } pub fn get_blinded_block( @@ -1936,6 +1935,15 @@ impl BeaconChain { }) } + pub fn verify_blob_sidecar_for_gossip( + self: &Arc, + blob_sidecar: SignedBlobSidecar, + subnet_id: u64, + ) -> Result, BlobError> // TODO(pawan): make a GossipVerifedBlob type + { + blob_verification::validate_blob_sidecar_for_gossip(blob_sidecar, subnet_id, self) + } + /// Accepts some 'LightClientOptimisticUpdate' from the network and attempts to verify it pub fn verify_optimistic_update_for_gossip( self: &Arc, @@ -2686,13 +2694,29 @@ impl BeaconChain { .map_err(BeaconChainError::TokioJoin)? } + pub async fn process_blob( + self: &Arc, + blob: GossipVerifiedBlob, + count_unrealized: CountUnrealized, + ) -> Result> { + self.check_availability_and_maybe_import( + |chain| chain.data_availability_checker.put_gossip_blob(blob), + count_unrealized, + ) + .await + } + /// Returns `Ok(block_root)` if the given `unverified_block` was successfully verified and /// imported into the chain. /// + /// For post deneb blocks, this returns a `BlockError::AvailabilityPending` error + /// if the corresponding blobs are not in the required caches. + /// /// Items that implement `IntoExecutionPendingBlock` include: /// /// - `SignedBeaconBlock` /// - `GossipVerifiedBlock` + /// - `BlockWrapper` /// /// ## Errors /// @@ -2704,105 +2728,67 @@ impl BeaconChain { unverified_block: B, count_unrealized: CountUnrealized, notify_execution_layer: NotifyExecutionLayer, - ) -> Result> { + ) -> Result> { // Start the Prometheus timer. let _full_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_TIMES); // Increment the Prometheus counter for block processing requests. metrics::inc_counter(&metrics::BLOCK_PROCESSING_REQUESTS); - let slot = unverified_block.block().slot(); - - // A small closure to group the verification and import errors. let chain = self.clone(); - let import_block = async move { - let execution_pending = unverified_block.into_execution_pending_block( - block_root, - &chain, - notify_execution_layer, - )?; - chain - .import_execution_pending_block(execution_pending, count_unrealized) - .await - }; - // Verify and import the block. - match import_block.await { - // The block was successfully verified and imported. Yay. - Ok(block_root) => { - trace!( - self.log, - "Beacon block imported"; - "block_root" => ?block_root, - "block_slot" => slot, - ); + let execution_pending = unverified_block.into_execution_pending_block( + block_root, + &chain, + notify_execution_layer, + )?; - // Increment the Prometheus counter for block processing successes. - metrics::inc_counter(&metrics::BLOCK_PROCESSING_SUCCESSES); + let executed_block = self + .clone() + .into_executed_block(execution_pending) + .await + .map_err(|e| self.handle_block_error(e))?; - Ok(block_root) - } - Err(e @ BlockError::BeaconChainError(BeaconChainError::TokioJoin(_))) => { - debug!( - self.log, - "Beacon block processing cancelled"; - "error" => ?e, - ); - Err(e) - } - // There was an error whilst attempting to verify and import the block. The block might - // be partially verified or partially imported. - Err(BlockError::BeaconChainError(e)) => { - crit!( - self.log, - "Beacon block processing error"; - "error" => ?e, - ); - Err(BlockError::BeaconChainError(e)) + match executed_block { + ExecutedBlock::Available(block) => { + self.import_available_block(Box::new(block), count_unrealized) + .await } - // The block failed verification. - Err(other) => { - trace!( - self.log, - "Beacon block rejected"; - "reason" => other.to_string(), - ); - Err(other) + ExecutedBlock::AvailabilityPending(block) => { + self.check_availability_and_maybe_import( + |chain| { + chain + .data_availability_checker + .put_pending_executed_block(block) + }, + count_unrealized, + ) + .await } } } - /// Accepts a fully-verified block and imports it into the chain without performing any - /// additional verification. + /// Accepts a fully-verified block and awaits on it's payload verification handle to + /// get a fully `ExecutedBlock` /// - /// An error is returned if the block was unable to be imported. It may be partially imported - /// (i.e., this function is not atomic). - async fn import_execution_pending_block( + /// An error is returned if the verification handle couldn't be awaited. + async fn into_executed_block( self: Arc, execution_pending_block: ExecutionPendingBlock, - count_unrealized: CountUnrealized, - ) -> Result> { + ) -> Result, BlockError> { let ExecutionPendingBlock { block, - block_root, - state, - parent_block, - confirmed_state_roots, + import_data, payload_verification_handle, - parent_eth1_finalization_data, - consensus_context, } = execution_pending_block; - let PayloadVerificationOutcome { - payload_verification_status, - is_valid_merge_transition_block, - } = payload_verification_handle + let payload_verification_outcome = payload_verification_handle .await .map_err(BeaconChainError::TokioJoin)? .ok_or(BeaconChainError::RuntimeShutdown)??; // Log the PoS pandas if a merge transition just occurred. - if is_valid_merge_transition_block { + if payload_verification_outcome.is_valid_merge_transition_block { info!(self.log, "{}", POS_PANDA_BANNER); info!( self.log, @@ -2830,9 +2816,91 @@ impl BeaconChain { .into_root() ); } + Ok(ExecutedBlock::new( + block, + import_data, + payload_verification_outcome, + )) + } + fn handle_block_error(&self, e: BlockError) -> BlockError { + match e { + e @ BlockError::BeaconChainError(BeaconChainError::TokioJoin(_)) => { + debug!( + self.log, + "Beacon block processing cancelled"; + "error" => ?e, + ); + e + } + BlockError::BeaconChainError(e) => { + crit!( + self.log, + "Beacon block processing error"; + "error" => ?e, + ); + BlockError::BeaconChainError(e) + } + other => { + trace!( + self.log, + "Beacon block rejected"; + "reason" => other.to_string(), + ); + other + } + } + } + + /// Accepts a fully-verified, available block and imports it into the chain without performing any + /// additional verification. + /// + /// An error is returned if the block was unable to be imported. It may be partially imported + /// (i.e., this function is not atomic). + pub async fn check_availability_and_maybe_import( + self: &Arc, + cache_fn: impl FnOnce(Arc) -> Result, AvailabilityCheckError>, + count_unrealized: CountUnrealized, + ) -> Result> { + let availability = cache_fn(self.clone())?; + match availability { + Availability::Available(block) => { + self.import_available_block(block, count_unrealized).await + } + Availability::PendingBlock(block_root) => { + Ok(AvailabilityProcessingStatus::PendingBlock(block_root)) + } + Availability::PendingBlobs(blob_ids) => { + Ok(AvailabilityProcessingStatus::PendingBlobs(blob_ids)) + } + } + } + + async fn import_available_block( + self: &Arc, + block: Box>, + count_unrealized: CountUnrealized, + ) -> Result> { + let AvailableExecutedBlock { + block, + import_data, + payload_verification_outcome, + } = *block; + + let BlockImportData { + block_root, + state, + parent_block, + parent_eth1_finalization_data, + confirmed_state_roots, + consensus_context, + } = import_data; + + let slot = block.slot(); + + // import let chain = self.clone(); - let block_hash = self + let result = self .spawn_blocking_handle( move || { chain.import_block( @@ -2840,7 +2908,7 @@ impl BeaconChain { block_root, state, confirmed_state_roots, - payload_verification_status, + payload_verification_outcome.payload_verification_status, count_unrealized, parent_block, parent_eth1_finalization_data, @@ -2849,12 +2917,32 @@ impl BeaconChain { }, "payload_verification_handle", ) - .await??; + .await + .map_err(|e| { + let b = BlockError::from(e); + self.handle_block_error(b) + })?; + + match result { + // The block was successfully verified and imported. Yay. + Ok(block_root) => { + trace!( + self.log, + "Beacon block imported"; + "block_root" => ?block_root, + "block_slot" => slot, + ); + + // Increment the Prometheus counter for block processing successes. + metrics::inc_counter(&metrics::BLOCK_PROCESSING_SUCCESSES); - Ok(block_hash) + Ok(AvailabilityProcessingStatus::Imported(block_root)) + } + Err(e) => Err(self.handle_block_error(e)), + } } - /// Accepts a fully-verified block and imports it into the chain without performing any + /// Accepts a fully-verified and available block and imports it into the chain without performing any /// additional verification. /// /// An error is returned if the block was unable to be imported. It may be partially imported @@ -3038,27 +3126,17 @@ impl BeaconChain { ops.push(StoreOp::PutBlock(block_root, signed_block.clone())); ops.push(StoreOp::PutState(block.state_root(), &state)); - // Only consider blobs if the eip4844 fork is enabled. - if let Some(data_availability_boundary) = self.data_availability_boundary() { - let block_epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); - let margin_epochs = self.store.get_config().blob_prune_margin_epochs; - let import_boundary = data_availability_boundary - margin_epochs; - - // Only store blobs at the data availability boundary, minus any configured epochs - // margin, or younger (of higher epoch number). - if block_epoch >= import_boundary { - if let Some(blobs) = blobs { - if !blobs.is_empty() { - //FIXME(sean) using this for debugging for now - info!( - self.log, "Writing blobs to store"; - "block_root" => ?block_root - ); - ops.push(StoreOp::PutBlobs(block_root, blobs)); - } - } + if let Some(blobs) = blobs { + if !blobs.is_empty() { + //FIXME(sean) using this for debugging for now + info!( + self.log, "Writing blobs to store"; + "block_root" => ?block_root + ); + ops.push(StoreOp::PutBlobs(block_root, blobs)); } } + let txn_lock = self.store.hot_db.begin_rw_transaction(); if let Err(e) = self.store.do_atomically_with_block_and_blobs_cache(ops) { @@ -4782,12 +4860,11 @@ impl BeaconChain { .as_ref() .ok_or(BlockProductionError::TrustedSetupNotInitialized)?; let beacon_block_root = block.canonical_root(); - let expected_kzg_commitments: &KzgCommitments = - block.body().blob_kzg_commitments().map_err(|_| { - BlockProductionError::InvalidBlockVariant( - "EIP4844 block does not contain kzg commitments".to_string(), - ) - })?; + let expected_kzg_commitments = block.body().blob_kzg_commitments().map_err(|_| { + BlockProductionError::InvalidBlockVariant( + "EIP4844 block does not contain kzg commitments".to_string(), + ) + })?; if expected_kzg_commitments.len() != blobs.len() { return Err(BlockProductionError::MissingKzgCommitment(format!( @@ -4836,7 +4913,8 @@ impl BeaconChain { .collect::, BlockProductionError>>()?, ); - self.blob_cache.put(beacon_block_root, blob_sidecars); + self.proposal_blob_cache + .put(beacon_block_root, blob_sidecars); } metrics::inc_counter(&metrics::BLOCK_PRODUCTION_SUCCESSES); @@ -4854,8 +4932,8 @@ impl BeaconChain { fn compute_blob_kzg_proofs( kzg: &Arc, - blobs: &Blobs<::EthSpec>, - expected_kzg_commitments: &KzgCommitments<::EthSpec>, + blobs: &Blobs, + expected_kzg_commitments: &KzgCommitments, slot: Slot, ) -> Result, BlockProductionError> { blobs @@ -6119,18 +6197,10 @@ impl BeaconChain { }) } - /// The epoch that is a data availability boundary, or the latest finalized epoch. - /// `None` if the `Eip4844` fork is disabled. - pub fn finalized_data_availability_boundary(&self) -> Option { - self.data_availability_boundary().map(|boundary| { - std::cmp::max( - boundary, - self.canonical_head - .cached_head() - .finalized_checkpoint() - .epoch, - ) - }) + /// Returns true if the given epoch lies within the da boundary and false otherwise. + pub fn block_needs_da_check(&self, block_epoch: Epoch) -> bool { + self.data_availability_boundary() + .map_or(false, |da_epoch| block_epoch >= da_epoch) } /// Returns `true` if we are at or past the `Eip4844` fork. This will always return `false` if diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index e2288dbb808..9d1a7c708ec 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -2,16 +2,20 @@ use derivative::Derivative; use slot_clock::SlotClock; use std::sync::Arc; -use crate::beacon_chain::{BeaconChain, BeaconChainTypes, MAXIMUM_GOSSIP_CLOCK_DISPARITY}; +use crate::beacon_chain::{ + BeaconChain, BeaconChainTypes, MAXIMUM_GOSSIP_CLOCK_DISPARITY, + VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT, +}; +use crate::data_availability_checker::{ + AvailabilityCheckError, AvailabilityPendingBlock, AvailableBlock, +}; +use crate::kzg_utils::{validate_blob, validate_blobs}; use crate::BeaconChainError; -use state_processing::per_block_processing::eip4844::eip4844::verify_kzg_commitments_against_transactions; -use types::signed_beacon_block::BlobReconstructionError; +use kzg::Kzg; use types::{ - BeaconBlockRef, BeaconStateError, BlobsSidecar, EthSpec, Hash256, KzgCommitment, - SignedBeaconBlock, SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockHeader, Slot, - Transactions, + BeaconBlockRef, BeaconStateError, BlobSidecar, BlobSidecarList, Epoch, EthSpec, Hash256, + KzgCommitment, SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlobSidecar, Slot, }; -use types::{Epoch, ExecPayload}; #[derive(Debug)] pub enum BlobError { @@ -62,15 +66,54 @@ pub enum BlobError { UnavailableBlobs, /// Blobs provided for a pre-Eip4844 fork. InconsistentFork, -} -impl From for BlobError { - fn from(e: BlobReconstructionError) -> Self { - match e { - BlobReconstructionError::UnavailableBlobs => BlobError::UnavailableBlobs, - BlobReconstructionError::InconsistentFork => BlobError::InconsistentFork, - } - } + /// The `blobs_sidecar.message.beacon_block_root` block is unknown. + /// + /// ## Peer scoring + /// + /// The blob points to a block we have not yet imported. The blob cannot be imported + /// into fork choice yet + UnknownHeadBlock { + beacon_block_root: Hash256, + }, + + /// The `BlobSidecar` was gossiped over an incorrect subnet. + InvalidSubnet { + expected: u64, + received: u64, + }, + + /// The sidecar corresponds to a slot older than the finalized head slot. + PastFinalizedSlot { + blob_slot: Slot, + finalized_slot: Slot, + }, + + /// The proposer index specified in the sidecar does not match the locally computed + /// proposer index. + ProposerIndexMismatch { + sidecar: usize, + local: usize, + }, + + ProposerSignatureInvalid, + + /// A sidecar with same slot, beacon_block_root and proposer_index but different blob is received for + /// the same blob index. + RepeatSidecar { + proposer: usize, + slot: Slot, + blob_index: usize, + }, + + /// The proposal_index corresponding to blob.beacon_block_root is not known. + /// + /// ## Peer scoring + /// + /// The block is invalid and the peer is faulty. + UnknownValidator(u64), + + BlobCacheError(AvailabilityCheckError), } impl From for BlobError { @@ -85,302 +128,220 @@ impl From for BlobError { } } -pub fn validate_blob_for_gossip( - block_wrapper: BlockWrapper, - block_root: Hash256, - chain: &BeaconChain, -) -> Result, BlobError> { - if let BlockWrapper::BlockAndBlob(ref block, ref blobs_sidecar) = block_wrapper { - let blob_slot = blobs_sidecar.beacon_block_slot; - // Do not gossip or process blobs from future or past slots. - let latest_permissible_slot = chain - .slot_clock - .now_with_future_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY) - .ok_or(BeaconChainError::UnableToReadSlot)?; - if blob_slot > latest_permissible_slot { - return Err(BlobError::FutureSlot { - message_slot: latest_permissible_slot, - latest_permissible_slot: blob_slot, - }); - } +/// A wrapper around a `BlobSidecar` that indicates it has been approved for re-gossiping on +/// the p2p network. +#[derive(Debug)] +pub struct GossipVerifiedBlob { + blob: Arc>, +} - if blob_slot != block.slot() { - return Err(BlobError::SlotMismatch { - blob_slot, - block_slot: block.slot(), - }); - } +impl GossipVerifiedBlob { + pub fn block_root(&self) -> Hash256 { + self.blob.block_root } - - block_wrapper.into_available_block(block_root, chain) } -fn verify_data_availability( - _blob_sidecar: &BlobsSidecar, - kzg_commitments: &[KzgCommitment], - transactions: &Transactions, - _block_slot: Slot, - _block_root: Hash256, +pub fn validate_blob_sidecar_for_gossip( + signed_blob_sidecar: SignedBlobSidecar, + subnet: u64, chain: &BeaconChain, -) -> Result<(), BlobError> { - if verify_kzg_commitments_against_transactions::(transactions, kzg_commitments) - .is_err() +) -> Result, BlobError> { + let blob_slot = signed_blob_sidecar.message.slot; + let blob_index = signed_blob_sidecar.message.index; + let block_root = signed_blob_sidecar.message.block_root; + + // Verify that the blob_sidecar was received on the correct subnet. + if blob_index != subnet { + return Err(BlobError::InvalidSubnet { + expected: blob_index, + received: subnet, + }); + } + + // Verify that the sidecar is not from a future slot. + let latest_permissible_slot = chain + .slot_clock + .now_with_future_tolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY) + .ok_or(BeaconChainError::UnableToReadSlot)?; + if blob_slot > latest_permissible_slot { + return Err(BlobError::FutureSlot { + message_slot: blob_slot, + latest_permissible_slot, + }); + } + + // TODO(pawan): Verify not from a past slot? + + // Verify that the sidecar slot is greater than the latest finalized slot + let latest_finalized_slot = chain + .head() + .finalized_checkpoint() + .epoch + .start_slot(T::EthSpec::slots_per_epoch()); + if blob_slot <= latest_finalized_slot { + return Err(BlobError::PastFinalizedSlot { + blob_slot, + finalized_slot: latest_finalized_slot, + }); + } + + // TODO(pawan): should we verify locally that the parent root is correct + // or just use whatever the proposer gives us? + let proposer_shuffling_root = signed_blob_sidecar.message.block_parent_root; + + let (proposer_index, fork) = match chain + .beacon_proposer_cache + .lock() + .get_slot::(proposer_shuffling_root, blob_slot) { - return Err(BlobError::TransactionCommitmentMismatch); - } - - // Validatate that the kzg proof is valid against the commitments and blobs - let _kzg = chain - .kzg - .as_ref() - .ok_or(BlobError::TrustedSetupNotInitialized)?; - - todo!("use `kzg_utils::validate_blobs` once the function is updated") - // if !kzg_utils::validate_blobs_sidecar( - // kzg, - // block_slot, - // block_root, - // kzg_commitments, - // blob_sidecar, - // ) - // .map_err(BlobError::KzgError)? - // { - // return Err(BlobError::InvalidKzgProof); - // } - // Ok(()) + Some(proposer) => (proposer.index, proposer.fork), + None => { + let state = &chain.canonical_head.cached_head().snapshot.beacon_state; + ( + state.get_beacon_proposer_index(blob_slot, &chain.spec)?, + state.fork(), + ) + } + }; + + let blob_proposer_index = signed_blob_sidecar.message.proposer_index; + if proposer_index != blob_proposer_index as usize { + return Err(BlobError::ProposerIndexMismatch { + sidecar: blob_proposer_index as usize, + local: proposer_index, + }); + } + + let signature_is_valid = { + let pubkey_cache = chain + .validator_pubkey_cache + .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT) + .ok_or(BeaconChainError::ValidatorPubkeyCacheLockTimeout) + .map_err(BlobError::BeaconChainError)?; + + let pubkey = pubkey_cache + .get(proposer_index) + .ok_or_else(|| BlobError::UnknownValidator(proposer_index as u64))?; + + signed_blob_sidecar.verify_signature( + None, + pubkey, + &fork, + chain.genesis_validators_root, + &chain.spec, + ) + }; + + if !signature_is_valid { + return Err(BlobError::ProposerSignatureInvalid); + } + + // TODO(pawan): kzg validations. + + // TODO(pawan): Check if other blobs for the same proposer index and blob index have been + // received and drop if required. + + // Verify if the corresponding block for this blob has been received. + // Note: this should be the last gossip check so that we can forward the blob + // over the gossip network even if we haven't received the corresponding block yet + // as all other validations have passed. + let block_opt = chain + .canonical_head + .fork_choice_read_lock() + .get_block(&block_root) + .or_else(|| chain.early_attester_cache.get_proto_block(block_root)); // TODO(pawan): should we be checking this cache? + + // TODO(pawan): this may be redundant with the new `AvailabilityProcessingStatus::PendingBlock variant` + if block_opt.is_none() { + return Err(BlobError::UnknownHeadBlock { + beacon_block_root: block_root, + }); + } + + Ok(GossipVerifiedBlob { + blob: signed_blob_sidecar.message, + }) } -/// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobsSidecar`]. This makes no -/// claims about data availability and should not be used in consensus. This struct is useful in -/// networking when we want to send blocks around without consensus checks. -#[derive(Clone, Debug, Derivative)] -#[derivative(PartialEq, Hash(bound = "E: EthSpec"))] -pub enum BlockWrapper { - Block(Arc>), - BlockAndBlob(Arc>, Arc>), +#[derive(Debug, Clone)] +pub struct KzgVerifiedBlob { + blob: Arc>, } -impl BlockWrapper { - pub fn new( - block: Arc>, - blobs_sidecar: Option>>, - ) -> Self { - if let Some(blobs_sidecar) = blobs_sidecar { - BlockWrapper::BlockAndBlob(block, blobs_sidecar) - } else { - BlockWrapper::Block(block) - } +impl KzgVerifiedBlob { + pub fn to_blob(self) -> Arc> { + self.blob } -} - -impl From> for BlockWrapper { - fn from(block: SignedBeaconBlock) -> Self { - BlockWrapper::Block(Arc::new(block)) + pub fn as_blob(&self) -> &BlobSidecar { + &self.blob } -} - -impl From> for BlockWrapper { - fn from(block: SignedBeaconBlockAndBlobsSidecar) -> Self { - let SignedBeaconBlockAndBlobsSidecar { - beacon_block, - blobs_sidecar, - } = block; - BlockWrapper::BlockAndBlob(beacon_block, blobs_sidecar) + pub fn clone_blob(&self) -> Arc> { + self.blob.clone() } -} - -impl From>> for BlockWrapper { - fn from(block: Arc>) -> Self { - BlockWrapper::Block(block) + pub fn kzg_commitment(&self) -> KzgCommitment { + self.blob.kzg_commitment } -} - -#[derive(Copy, Clone)] -pub enum DataAvailabilityCheckRequired { - Yes, - No, -} - -pub trait IntoAvailableBlock { - fn into_available_block( - self, - block_root: Hash256, - chain: &BeaconChain, - ) -> Result, BlobError>; -} - -impl IntoAvailableBlock for BlockWrapper { - fn into_available_block( - self, - block_root: Hash256, - chain: &BeaconChain, - ) -> Result, BlobError> { - let data_availability_boundary = chain.data_availability_boundary(); - let da_check_required = - data_availability_boundary.map_or(DataAvailabilityCheckRequired::No, |boundary| { - if self.slot().epoch(T::EthSpec::slots_per_epoch()) >= boundary { - DataAvailabilityCheckRequired::Yes - } else { - DataAvailabilityCheckRequired::No - } - }); - match self { - BlockWrapper::Block(block) => AvailableBlock::new(block, block_root, da_check_required), - BlockWrapper::BlockAndBlob(block, blobs_sidecar) => { - if matches!(da_check_required, DataAvailabilityCheckRequired::Yes) { - let kzg_commitments = block - .message() - .body() - .blob_kzg_commitments() - .map_err(|_| BlobError::KzgCommitmentMissing)?; - let transactions = block - .message() - .body() - .execution_payload_eip4844() - .map(|payload| payload.transactions()) - .map_err(|_| BlobError::TransactionsMissing)? - .ok_or(BlobError::TransactionsMissing)?; - verify_data_availability( - &blobs_sidecar, - kzg_commitments, - transactions, - block.slot(), - block_root, - chain, - )?; - } - - AvailableBlock::new_with_blobs(block, blobs_sidecar, da_check_required) - } - } + pub fn block_root(&self) -> Hash256 { + self.blob.block_root } } -/// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobsSidecar`]. An -/// `AvailableBlock` has passed any required data availability checks and should be used in -/// consensus. This newtype wraps `AvailableBlockInner` to ensure data availability checks -/// cannot be circumvented on construction. -#[derive(Clone, Debug, Derivative)] -#[derivative(PartialEq, Hash(bound = "E: EthSpec"))] -pub struct AvailableBlock(AvailableBlockInner); - -/// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobsSidecar`]. -#[derive(Clone, Debug, Derivative)] -#[derivative(PartialEq, Hash(bound = "E: EthSpec"))] -enum AvailableBlockInner { - Block(Arc>), - BlockAndBlob(SignedBeaconBlockAndBlobsSidecar), -} - -impl AvailableBlock { - pub fn new( - beacon_block: Arc>, - block_root: Hash256, - da_check_required: DataAvailabilityCheckRequired, - ) -> Result { - match beacon_block.as_ref() { - // No data availability check required prior to Eip4844. - SignedBeaconBlock::Base(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Capella(_) - | SignedBeaconBlock::Merge(_) => { - Ok(AvailableBlock(AvailableBlockInner::Block(beacon_block))) - } - SignedBeaconBlock::Eip4844(_) => { - match da_check_required { - DataAvailabilityCheckRequired::Yes => { - // Attempt to reconstruct empty blobs here. - let blobs_sidecar = beacon_block - .reconstruct_empty_blobs(Some(block_root)) - .map(Arc::new)?; - Ok(AvailableBlock(AvailableBlockInner::BlockAndBlob( - SignedBeaconBlockAndBlobsSidecar { - beacon_block, - blobs_sidecar, - }, - ))) - } - DataAvailabilityCheckRequired::No => { - Ok(AvailableBlock(AvailableBlockInner::Block(beacon_block))) - } - } - } - } - } - - /// This function is private because an `AvailableBlock` should be - /// constructed via the `into_available_block` method. - fn new_with_blobs( - beacon_block: Arc>, - blobs_sidecar: Arc>, - da_check_required: DataAvailabilityCheckRequired, - ) -> Result { - match beacon_block.as_ref() { - // This method shouldn't be called with a pre-Eip4844 block. - SignedBeaconBlock::Base(_) - | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Capella(_) - | SignedBeaconBlock::Merge(_) => Err(BlobError::InconsistentFork), - SignedBeaconBlock::Eip4844(_) => { - match da_check_required { - DataAvailabilityCheckRequired::Yes => Ok(AvailableBlock( - AvailableBlockInner::BlockAndBlob(SignedBeaconBlockAndBlobsSidecar { - beacon_block, - blobs_sidecar, - }), - )), - DataAvailabilityCheckRequired::No => { - // Blobs were not verified so we drop them, we'll instead just pass around - // an available `Eip4844` block without blobs. - Ok(AvailableBlock(AvailableBlockInner::Block(beacon_block))) - } - } - } - } - } - - pub fn blobs(&self) -> Option>> { - match &self.0 { - AvailableBlockInner::Block(_) => None, - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - Some(block_sidecar_pair.blobs_sidecar.clone()) - } - } - } - - pub fn deconstruct(self) -> (Arc>, Option>>) { - match self.0 { - AvailableBlockInner::Block(block) => (block, None), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - let SignedBeaconBlockAndBlobsSidecar { - beacon_block, - blobs_sidecar, - } = block_sidecar_pair; - (beacon_block, Some(blobs_sidecar)) - } - } +pub fn verify_kzg_for_blob( + blob: GossipVerifiedBlob, + kzg: &Kzg, +) -> Result, AvailabilityCheckError> { + //TODO(sean) remove clone + if validate_blob::( + kzg, + blob.blob.blob.clone(), + blob.blob.kzg_commitment, + blob.blob.kzg_proof, + ) + .map_err(AvailabilityCheckError::Kzg)? + { + Ok(KzgVerifiedBlob { blob: blob.blob }) + } else { + Err(AvailabilityCheckError::KzgVerificationFailed) } } -pub trait IntoBlockWrapper: AsBlock { - fn into_block_wrapper(self) -> BlockWrapper; -} - -impl IntoBlockWrapper for BlockWrapper { - fn into_block_wrapper(self) -> BlockWrapper { - self +pub fn verify_kzg_for_blob_list( + blob_list: BlobSidecarList, + kzg: &Kzg, +) -> Result, AvailabilityCheckError> { + let (blobs, (commitments, proofs)): (Vec<_>, (Vec<_>, Vec<_>)) = blob_list + .clone() + .into_iter() + //TODO(sean) remove clone + .map(|blob| (blob.blob.clone(), (blob.kzg_commitment, blob.kzg_proof))) + .unzip(); + if validate_blobs::( + kzg, + commitments.as_slice(), + blobs.as_slice(), + proofs.as_slice(), + ) + .map_err(AvailabilityCheckError::Kzg)? + { + Ok(blob_list + .into_iter() + .map(|blob| KzgVerifiedBlob { blob }) + .collect()) + } else { + Err(AvailabilityCheckError::KzgVerificationFailed) } } -impl IntoBlockWrapper for AvailableBlock { - fn into_block_wrapper(self) -> BlockWrapper { - let (block, blobs) = self.deconstruct(); - if let Some(blobs) = blobs { - BlockWrapper::BlockAndBlob(block, blobs) - } else { - BlockWrapper::Block(block) - } - } +pub type KzgVerifiedBlobList = Vec>; + +#[derive(Debug, Clone)] +pub enum MaybeAvailableBlock { + /// This variant is fully available. + /// i.e. for pre-eip4844 blocks, it contains a (`SignedBeaconBlock`, `Blobs::None`) and for + /// post-4844 blocks, it contains a `SignedBeaconBlock` and a Blobs variant other than `Blobs::None`. + Available(AvailableBlock), + /// This variant is not fully available and requires blobs to become fully available. + AvailabilityPending(AvailabilityPendingBlock), } pub trait AsBlock { @@ -393,193 +354,149 @@ pub trait AsBlock { fn as_block(&self) -> &SignedBeaconBlock; fn block_cloned(&self) -> Arc>; fn canonical_root(&self) -> Hash256; + fn into_block_wrapper(self) -> BlockWrapper; } -impl AsBlock for BlockWrapper { +impl AsBlock for MaybeAvailableBlock { fn slot(&self) -> Slot { - match self { - BlockWrapper::Block(block) => block.slot(), - BlockWrapper::BlockAndBlob(block, _) => block.slot(), - } + self.as_block().slot() } fn epoch(&self) -> Epoch { - match self { - BlockWrapper::Block(block) => block.epoch(), - BlockWrapper::BlockAndBlob(block, _) => block.epoch(), - } + self.as_block().epoch() } fn parent_root(&self) -> Hash256 { - match self { - BlockWrapper::Block(block) => block.parent_root(), - BlockWrapper::BlockAndBlob(block, _) => block.parent_root(), - } + self.as_block().parent_root() } fn state_root(&self) -> Hash256 { - match self { - BlockWrapper::Block(block) => block.state_root(), - BlockWrapper::BlockAndBlob(block, _) => block.state_root(), - } + self.as_block().state_root() } fn signed_block_header(&self) -> SignedBeaconBlockHeader { - match &self { - BlockWrapper::Block(block) => block.signed_block_header(), - BlockWrapper::BlockAndBlob(block, _) => block.signed_block_header(), - } + self.as_block().signed_block_header() } fn message(&self) -> BeaconBlockRef { - match &self { - BlockWrapper::Block(block) => block.message(), - BlockWrapper::BlockAndBlob(block, _) => block.message(), - } + self.as_block().message() } fn as_block(&self) -> &SignedBeaconBlock { match &self { - BlockWrapper::Block(block) => block, - BlockWrapper::BlockAndBlob(block, _) => block, + MaybeAvailableBlock::Available(block) => block.as_block(), + MaybeAvailableBlock::AvailabilityPending(block) => block.as_block(), } } fn block_cloned(&self) -> Arc> { match &self { - BlockWrapper::Block(block) => block.clone(), - BlockWrapper::BlockAndBlob(block, _) => block.clone(), + MaybeAvailableBlock::Available(block) => block.block_cloned(), + MaybeAvailableBlock::AvailabilityPending(block) => block.block_cloned(), } } fn canonical_root(&self) -> Hash256 { - match &self { - BlockWrapper::Block(block) => block.canonical_root(), - BlockWrapper::BlockAndBlob(block, _) => block.canonical_root(), + self.as_block().canonical_root() + } + + fn into_block_wrapper(self) -> BlockWrapper { + match self { + MaybeAvailableBlock::Available(available_block) => available_block.into_block_wrapper(), + MaybeAvailableBlock::AvailabilityPending(pending_block) => { + BlockWrapper::Block(pending_block.to_block()) + } } } } -impl AsBlock for &BlockWrapper { +impl AsBlock for &MaybeAvailableBlock { fn slot(&self) -> Slot { - match self { - BlockWrapper::Block(block) => block.slot(), - BlockWrapper::BlockAndBlob(block, _) => block.slot(), - } + self.as_block().slot() } fn epoch(&self) -> Epoch { - match self { - BlockWrapper::Block(block) => block.epoch(), - BlockWrapper::BlockAndBlob(block, _) => block.epoch(), - } + self.as_block().epoch() } fn parent_root(&self) -> Hash256 { - match self { - BlockWrapper::Block(block) => block.parent_root(), - BlockWrapper::BlockAndBlob(block, _) => block.parent_root(), - } + self.as_block().parent_root() } fn state_root(&self) -> Hash256 { - match self { - BlockWrapper::Block(block) => block.state_root(), - BlockWrapper::BlockAndBlob(block, _) => block.state_root(), - } + self.as_block().state_root() } fn signed_block_header(&self) -> SignedBeaconBlockHeader { - match &self { - BlockWrapper::Block(block) => block.signed_block_header(), - BlockWrapper::BlockAndBlob(block, _) => block.signed_block_header(), - } + self.as_block().signed_block_header() } fn message(&self) -> BeaconBlockRef { - match &self { - BlockWrapper::Block(block) => block.message(), - BlockWrapper::BlockAndBlob(block, _) => block.message(), - } + self.as_block().message() } fn as_block(&self) -> &SignedBeaconBlock { match &self { - BlockWrapper::Block(block) => block, - BlockWrapper::BlockAndBlob(block, _) => block, + MaybeAvailableBlock::Available(block) => block.as_block(), + MaybeAvailableBlock::AvailabilityPending(block) => block.as_block(), } } fn block_cloned(&self) -> Arc> { match &self { - BlockWrapper::Block(block) => block.clone(), - BlockWrapper::BlockAndBlob(block, _) => block.clone(), + MaybeAvailableBlock::Available(block) => block.block_cloned(), + MaybeAvailableBlock::AvailabilityPending(block) => block.block_cloned(), } } fn canonical_root(&self) -> Hash256 { - match &self { - BlockWrapper::Block(block) => block.canonical_root(), - BlockWrapper::BlockAndBlob(block, _) => block.canonical_root(), - } + self.as_block().canonical_root() + } + + fn into_block_wrapper(self) -> BlockWrapper { + self.clone().into_block_wrapper() } } -impl AsBlock for AvailableBlock { +#[derive(Debug, Clone, Derivative)] +#[derivative(Hash(bound = "E: EthSpec"))] +pub enum BlockWrapper { + Block(Arc>), + BlockAndBlobs(Arc>, BlobSidecarList), +} + +impl AsBlock for BlockWrapper { fn slot(&self) -> Slot { - match &self.0 { - AvailableBlockInner::Block(block) => block.slot(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.slot() - } - } + self.as_block().slot() } fn epoch(&self) -> Epoch { - match &self.0 { - AvailableBlockInner::Block(block) => block.epoch(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.epoch() - } - } + self.as_block().epoch() } fn parent_root(&self) -> Hash256 { - match &self.0 { - AvailableBlockInner::Block(block) => block.parent_root(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.parent_root() - } - } + self.as_block().parent_root() } fn state_root(&self) -> Hash256 { - match &self.0 { - AvailableBlockInner::Block(block) => block.state_root(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.state_root() - } - } + self.as_block().state_root() } fn signed_block_header(&self) -> SignedBeaconBlockHeader { - match &self.0 { - AvailableBlockInner::Block(block) => block.signed_block_header(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.signed_block_header() - } - } + self.as_block().signed_block_header() } fn message(&self) -> BeaconBlockRef { - match &self.0 { - AvailableBlockInner::Block(block) => block.message(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.message() - } - } + self.as_block().message() } fn as_block(&self) -> &SignedBeaconBlock { - match &self.0 { - AvailableBlockInner::Block(block) => block, - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - &block_sidecar_pair.beacon_block - } + match &self { + BlockWrapper::Block(block) => block, + BlockWrapper::BlockAndBlobs(block, _) => block, } } fn block_cloned(&self) -> Arc> { - match &self.0 { - AvailableBlockInner::Block(block) => block.clone(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.clone() - } + match &self { + BlockWrapper::Block(block) => block.clone(), + BlockWrapper::BlockAndBlobs(block, _) => block.clone(), } } fn canonical_root(&self) -> Hash256 { - match &self.0 { - AvailableBlockInner::Block(block) => block.canonical_root(), - AvailableBlockInner::BlockAndBlob(block_sidecar_pair) => { - block_sidecar_pair.beacon_block.canonical_root() - } - } + self.as_block().canonical_root() + } + + fn into_block_wrapper(self) -> BlockWrapper { + self + } +} + +impl From>> for BlockWrapper { + fn from(value: Arc>) -> Self { + Self::Block(value) + } +} + +impl From> for BlockWrapper { + fn from(value: SignedBeaconBlock) -> Self { + Self::Block(Arc::new(value)) } } diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index a1f7abf0676..4cadc86bd23 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -24,9 +24,6 @@ //! â–¼ //! SignedBeaconBlock //! | -//! â–¼ -//! AvailableBlock -//! | //! |--------------- //! | | //! | â–¼ @@ -51,9 +48,9 @@ // returned alongside. #![allow(clippy::result_large_err)] -use crate::blob_verification::{ - validate_blob_for_gossip, AsBlock, AvailableBlock, BlobError, BlockWrapper, IntoAvailableBlock, - IntoBlockWrapper, +use crate::blob_verification::{AsBlock, BlobError, BlockWrapper, MaybeAvailableBlock}; +use crate::data_availability_checker::{ + AvailabilityCheckError, AvailabilityPendingBlock, AvailableBlock, }; use crate::eth1_finalization_cache::Eth1FinalizationData; use crate::execution_payload::{ @@ -96,7 +93,6 @@ use std::time::Duration; use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp}; use task_executor::JoinHandle; use tree_hash::TreeHash; -use types::signed_beacon_block::BlobReconstructionError; use types::ExecPayload; use types::{ BeaconBlockRef, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, CloneConfig, Epoch, @@ -314,6 +310,7 @@ pub enum BlockError { parent_root: Hash256, }, BlobValidation(BlobError), + AvailabilityCheck(AvailabilityCheckError), } impl From for BlockError { @@ -322,6 +319,12 @@ impl From for BlockError { } } +impl From for BlockError { + fn from(e: AvailabilityCheckError) -> Self { + Self::AvailabilityCheck(e) + } +} + /// Returned when block validation failed due to some issue verifying /// the execution payload. #[derive(Debug)] @@ -494,13 +497,8 @@ impl From for BlockError { } } -impl From for BlockError { - fn from(e: BlobReconstructionError) -> Self { - BlockError::BlobValidation(BlobError::from(e)) - } -} - /// Stores information about verifying a payload against an execution engine. +#[derive(Clone)] pub struct PayloadVerificationOutcome { pub payload_verification_status: PayloadVerificationStatus, pub is_valid_merge_transition_block: bool, @@ -605,14 +603,14 @@ pub fn signature_verify_chain_segment( signature_verifier.include_all_signatures(block.as_block(), &mut consensus_context)?; - //FIXME(sean) batch kzg verification - let available_block = block.clone().into_available_block(*block_root, chain)?; - consensus_context = consensus_context.set_kzg_commitments_consistent(true); + let maybe_available_block = chain + .data_availability_checker + .check_availability(block.clone())?; // Save the block and its consensus context. The context will have had its proposer index // and attesting indices filled in, which can be used to accelerate later block processing. signature_verified_blocks.push(SignatureVerifiedBlock { - block: available_block, + block: maybe_available_block, block_root: *block_root, parent: None, consensus_context, @@ -637,7 +635,7 @@ pub fn signature_verify_chain_segment( #[derive(Derivative)] #[derivative(Debug(bound = "T: BeaconChainTypes"))] pub struct GossipVerifiedBlock { - pub block: AvailableBlock, + pub block: MaybeAvailableBlock, pub block_root: Hash256, parent: Option>, consensus_context: ConsensusContext, @@ -646,7 +644,7 @@ pub struct GossipVerifiedBlock { /// A wrapper around a `SignedBeaconBlock` that indicates that all signatures (except the deposit /// signatures) have been verified. pub struct SignatureVerifiedBlock { - block: AvailableBlock, + block: MaybeAvailableBlock, block_root: Hash256, parent: Option>, consensus_context: ConsensusContext, @@ -669,14 +667,103 @@ type PayloadVerificationHandle = /// due to finality or some other event. A `ExecutionPendingBlock` should be imported into the /// `BeaconChain` immediately after it is instantiated. pub struct ExecutionPendingBlock { - pub block: AvailableBlock, + pub block: MaybeAvailableBlock, + pub import_data: BlockImportData, + pub payload_verification_handle: PayloadVerificationHandle, +} + +pub enum ExecutedBlock { + Available(AvailableExecutedBlock), + AvailabilityPending(AvailabilityPendingExecutedBlock), +} + +impl ExecutedBlock { + pub fn as_block(&self) -> &SignedBeaconBlock { + match self { + Self::Available(available) => available.block.block(), + Self::AvailabilityPending(pending) => pending.block.as_block(), + } + } +} + +impl std::fmt::Debug for ExecutedBlock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.as_block()) + } +} + +impl ExecutedBlock { + pub fn new( + block: MaybeAvailableBlock, + import_data: BlockImportData, + payload_verification_outcome: PayloadVerificationOutcome, + ) -> Self { + match block { + MaybeAvailableBlock::Available(available_block) => { + Self::Available(AvailableExecutedBlock::new( + available_block, + import_data, + payload_verification_outcome, + )) + } + MaybeAvailableBlock::AvailabilityPending(pending_block) => { + Self::AvailabilityPending(AvailabilityPendingExecutedBlock::new( + pending_block, + import_data, + payload_verification_outcome, + )) + } + } + } +} + +pub struct AvailableExecutedBlock { + pub block: AvailableBlock, + pub import_data: BlockImportData, + pub payload_verification_outcome: PayloadVerificationOutcome, +} + +impl AvailableExecutedBlock { + pub fn new( + block: AvailableBlock, + import_data: BlockImportData, + payload_verification_outcome: PayloadVerificationOutcome, + ) -> Self { + Self { + block, + import_data, + payload_verification_outcome, + } + } +} + +pub struct AvailabilityPendingExecutedBlock { + pub block: AvailabilityPendingBlock, + pub import_data: BlockImportData, + pub payload_verification_outcome: PayloadVerificationOutcome, +} + +impl AvailabilityPendingExecutedBlock { + pub fn new( + block: AvailabilityPendingBlock, + import_data: BlockImportData, + payload_verification_outcome: PayloadVerificationOutcome, + ) -> Self { + Self { + block, + import_data, + payload_verification_outcome, + } + } +} + +pub struct BlockImportData { pub block_root: Hash256, - pub state: BeaconState, - pub parent_block: SignedBeaconBlock>, + pub state: BeaconState, + pub parent_block: SignedBeaconBlock>, pub parent_eth1_finalization_data: Eth1FinalizationData, pub confirmed_state_roots: Vec, - pub consensus_context: ConsensusContext, - pub payload_verification_handle: PayloadVerificationHandle, + pub consensus_context: ConsensusContext, } /// Implemented on types that can be converted into a `ExecutionPendingBlock`. @@ -720,19 +807,20 @@ impl GossipVerifiedBlock { block: BlockWrapper, chain: &BeaconChain, ) -> Result> { + let maybe_available = chain.data_availability_checker.check_availability(block)?; // If the block is valid for gossip we don't supply it to the slasher here because // we assume it will be transformed into a fully verified block. We *do* need to supply // it to the slasher if an error occurs, because that's the end of this block's journey, // and it could be a repeat proposal (a likely cause for slashing!). - let header = block.signed_block_header(); - Self::new_without_slasher_checks(block, chain).map_err(|e| { + let header = maybe_available.signed_block_header(); + Self::new_without_slasher_checks(maybe_available, chain).map_err(|e| { process_block_slash_info(chain, BlockSlashInfo::from_early_error(header, e)) }) } /// As for new, but doesn't pass the block to the slasher. fn new_without_slasher_checks( - block: BlockWrapper, + block: MaybeAvailableBlock, chain: &BeaconChain, ) -> Result> { // Ensure the block is the correct structure for the fork at `block.slot()`. @@ -929,16 +1017,14 @@ impl GossipVerifiedBlock { // Validate the block's execution_payload (if any). validate_execution_payload_for_gossip(&parent_block, block.message(), chain)?; - let available_block = validate_blob_for_gossip(block, block_root, chain)?; - // Having checked the proposer index and the block root we can cache them. - let consensus_context = ConsensusContext::new(available_block.slot()) + let consensus_context = ConsensusContext::new(block.slot()) .set_current_block_root(block_root) - .set_proposer_index(available_block.as_block().message().proposer_index()) + .set_proposer_index(block.as_block().message().proposer_index()) .set_kzg_commitments_consistent(true); Ok(Self { - block: available_block, + block, block_root, parent, consensus_context, @@ -978,10 +1064,11 @@ impl SignatureVerifiedBlock { /// /// Returns an error if the block is invalid, or if the block was unable to be verified. pub fn new( - block: AvailableBlock, + block: BlockWrapper, block_root: Hash256, chain: &BeaconChain, ) -> Result> { + let block = chain.data_availability_checker.check_availability(block)?; // Ensure the block is the correct structure for the fork at `block.slot()`. block .as_block() @@ -1028,7 +1115,7 @@ impl SignatureVerifiedBlock { /// As for `new` above but producing `BlockSlashInfo`. pub fn check_slashable( - block: AvailableBlock, + block: BlockWrapper, block_root: Hash256, chain: &BeaconChain, ) -> Result>> { @@ -1137,12 +1224,7 @@ impl IntoExecutionPendingBlock for Arc IntoExecutionPendingBlock for Arc IntoExecutionPendingBlock for AvailableBlock { +impl IntoExecutionPendingBlock for BlockWrapper { /// Verifies the `SignedBeaconBlock` by first transforming it into a `SignatureVerifiedBlock` /// and then using that implementation of `IntoExecutionPendingBlock` to complete verification. fn into_execution_pending_block_slashable( @@ -1182,7 +1264,7 @@ impl ExecutionPendingBlock { /// /// Returns an error if the block is invalid, or if the block was unable to be verified. pub fn from_signature_verified_components( - block: AvailableBlock, + block: MaybeAvailableBlock, block_root: Hash256, parent: PreProcessingSnapshot, mut consensus_context: ConsensusContext, @@ -1562,12 +1644,14 @@ impl ExecutionPendingBlock { Ok(Self { block, - block_root, - state, - parent_block: parent.beacon_block, - parent_eth1_finalization_data, - confirmed_state_roots, - consensus_context, + import_data: BlockImportData { + block_root, + state, + parent_block: parent.beacon_block, + parent_eth1_finalization_data, + confirmed_state_roots, + consensus_context, + }, payload_verification_handle, }) } @@ -1648,7 +1732,7 @@ fn check_block_against_finalized_slot( /// Taking a lock on the `chain.canonical_head.fork_choice` might cause a deadlock here. pub fn check_block_is_finalized_checkpoint_or_descendant< T: BeaconChainTypes, - B: IntoBlockWrapper, + B: AsBlock, >( chain: &BeaconChain, fork_choice: &BeaconForkChoice, @@ -1746,8 +1830,8 @@ pub fn get_block_root(block: &SignedBeaconBlock) -> Hash256 { #[allow(clippy::type_complexity)] fn verify_parent_block_is_known( chain: &BeaconChain, - block: BlockWrapper, -) -> Result<(ProtoBlock, BlockWrapper), BlockError> { + block: MaybeAvailableBlock, +) -> Result<(ProtoBlock, MaybeAvailableBlock), BlockError> { if let Some(proto_block) = chain .canonical_head .fork_choice_read_lock() @@ -1755,7 +1839,7 @@ fn verify_parent_block_is_known( { Ok((proto_block, block)) } else { - Err(BlockError::ParentUnknown(block)) + Err(BlockError::ParentUnknown(block.into_block_wrapper())) } } @@ -1764,7 +1848,7 @@ fn verify_parent_block_is_known( /// Returns `Err(BlockError::ParentUnknown)` if the parent is not found, or if an error occurs /// whilst attempting the operation. #[allow(clippy::type_complexity)] -fn load_parent>( +fn load_parent>( block_root: Hash256, block: B, chain: &BeaconChain, diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 3698549ed44..7620a588d68 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1,5 +1,6 @@ use crate::beacon_chain::{CanonicalHead, BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, OP_POOL_DB_KEY}; use crate::blob_cache::BlobCache; +use crate::data_availability_checker::DataAvailabilityChecker; use crate::eth1_chain::{CachingEth1Backend, SszEth1}; use crate::eth1_finalization_cache::Eth1FinalizationCache; use crate::fork_choice_signal::ForkChoiceSignalTx; @@ -641,7 +642,8 @@ where let kzg = if let Some(trusted_setup) = self.trusted_setup { let kzg = Kzg::new_from_trusted_setup(trusted_setup) .map_err(|e| format!("Failed to load trusted setup: {:?}", e))?; - Some(Arc::new(kzg)) + let kzg_arc = Arc::new(kzg); + Some(kzg_arc) } else { None }; @@ -781,14 +783,14 @@ where let shuffling_cache_size = self.chain_config.shuffling_cache_size; let beacon_chain = BeaconChain { - spec: self.spec, + spec: self.spec.clone(), config: self.chain_config, store, task_executor: self .task_executor .ok_or("Cannot build without task executor")?, store_migrator, - slot_clock, + slot_clock: slot_clock.clone(), op_pool: self.op_pool.ok_or("Cannot build without op pool")?, // TODO: allow for persisting and loading the pool from disk. naive_aggregation_pool: <_>::default(), @@ -847,7 +849,13 @@ where graffiti: self.graffiti, slasher: self.slasher.clone(), validator_monitor: RwLock::new(validator_monitor), - blob_cache: BlobCache::default(), + //TODO(sean) should we move kzg solely to the da checker? + data_availability_checker: DataAvailabilityChecker::new( + slot_clock, + kzg.clone(), + self.spec, + ), + proposal_blob_cache: BlobCache::default(), kzg, }; diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs new file mode 100644 index 00000000000..42e97a974e4 --- /dev/null +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -0,0 +1,516 @@ +use crate::blob_verification::{ + verify_kzg_for_blob, verify_kzg_for_blob_list, AsBlock, BlockWrapper, GossipVerifiedBlob, + KzgVerifiedBlob, KzgVerifiedBlobList, MaybeAvailableBlock, +}; +use crate::block_verification::{AvailabilityPendingExecutedBlock, AvailableExecutedBlock}; + +use kzg::Error as KzgError; +use kzg::Kzg; +use parking_lot::{Mutex, RwLock}; +use slot_clock::SlotClock; +use ssz_types::{Error, VariableList}; +use state_processing::per_block_processing::eip4844::eip4844::verify_kzg_commitments_against_transactions; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::sync::Arc; +use types::beacon_block_body::KzgCommitments; +use types::blob_sidecar::{BlobIdentifier, BlobSidecar}; +use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; +use types::{ + BeaconBlockRef, BlobSidecarList, ChainSpec, Epoch, EthSpec, ExecPayload, FullPayload, Hash256, + SignedBeaconBlock, SignedBeaconBlockHeader, Slot, +}; + +#[derive(Debug)] +pub enum AvailabilityCheckError { + DuplicateBlob(Hash256), + Kzg(KzgError), + KzgVerificationFailed, + KzgNotInitialized, + SszTypes(ssz_types::Error), + MissingBlobs, + NumBlobsMismatch { + num_kzg_commitments: usize, + num_blobs: usize, + }, + TxKzgCommitmentMismatch, + KzgCommitmentMismatch { + blob_index: u64, + }, + Pending, + IncorrectFork, +} + +impl From for AvailabilityCheckError { + fn from(value: Error) -> Self { + Self::SszTypes(value) + } +} + +/// This cache contains +/// - blobs that have been gossip verified +/// - commitments for blocks that have been gossip verified, but the commitments themselves +/// have not been verified against blobs +/// - blocks that have been fully verified and only require a data availability check +pub struct DataAvailabilityChecker { + rpc_blob_cache: RwLock>>>, + gossip_blob_cache: Mutex>>, + slot_clock: S, + kzg: Option>, + spec: ChainSpec, +} + +struct GossipBlobCache { + verified_blobs: Vec>, + executed_block: Option>, +} + +pub enum Availability { + PendingBlobs(Vec), + PendingBlock(Hash256), + Available(Box>), +} + +impl DataAvailabilityChecker { + pub fn new(slot_clock: S, kzg: Option>, spec: ChainSpec) -> Self { + Self { + rpc_blob_cache: <_>::default(), + gossip_blob_cache: <_>::default(), + slot_clock, + kzg, + spec, + } + } + + /// This first validate the KZG commitments included in the blob sidecar. + /// Check if we've cached other blobs for this block. If it completes a set and we also + /// have a block cached, return the Availability variant triggering block import. + /// Otherwise cache the blob sidecar. + /// + /// This should only accept gossip verified blobs, so we should not have to worry about dupes. + pub fn put_gossip_blob( + &self, + verified_blob: GossipVerifiedBlob, + ) -> Result, AvailabilityCheckError> { + let block_root = verified_blob.block_root(); + + let kzg_verified_blob = if let Some(kzg) = self.kzg.as_ref() { + verify_kzg_for_blob(verified_blob, kzg)? + } else { + return Err(AvailabilityCheckError::KzgNotInitialized); + }; + + //TODO(sean) can we just use a referece to the blob here? + let blob = kzg_verified_blob.clone_blob(); + + // check if we have a block + // check if the complete set matches the block + // verify, otherwise cache + + let mut blob_cache = self.gossip_blob_cache.lock(); + + // Gossip cache. + let availability = match blob_cache.entry(blob.block_root) { + Entry::Occupied(mut occupied_entry) => { + // All blobs reaching this cache should be gossip verified and gossip verification + // should filter duplicates, as well as validate indices. + let cache = occupied_entry.get_mut(); + + cache + .verified_blobs + .insert(blob.index as usize, kzg_verified_blob); + + if let Some(executed_block) = cache.executed_block.take() { + self.check_block_availability_or_cache(cache, executed_block)? + } else { + Availability::PendingBlock(block_root) + } + } + Entry::Vacant(vacant_entry) => { + let block_root = kzg_verified_blob.block_root(); + vacant_entry.insert(GossipBlobCache { + verified_blobs: vec![kzg_verified_blob], + executed_block: None, + }); + Availability::PendingBlock(block_root) + } + }; + + drop(blob_cache); + + // RPC cache. + self.rpc_blob_cache.write().insert(blob.id(), blob.clone()); + + Ok(availability) + } + + /// Check if we have all the blobs for a block. If we do, return the Availability variant that + /// triggers import of the block. + pub fn put_pending_executed_block( + &self, + executed_block: AvailabilityPendingExecutedBlock, + ) -> Result, AvailabilityCheckError> { + let mut guard = self.gossip_blob_cache.lock(); + let entry = guard.entry(executed_block.import_data.block_root); + + let availability = match entry { + Entry::Occupied(mut occupied_entry) => { + let cache: &mut GossipBlobCache = occupied_entry.get_mut(); + + self.check_block_availability_or_cache(cache, executed_block)? + } + Entry::Vacant(vacant_entry) => { + let kzg_commitments_len = executed_block.block.kzg_commitments()?.len(); + let mut blob_ids = Vec::with_capacity(kzg_commitments_len); + for i in 0..kzg_commitments_len { + blob_ids.push(BlobIdentifier { + block_root: executed_block.import_data.block_root, + index: i as u64, + }); + } + + vacant_entry.insert(GossipBlobCache { + verified_blobs: vec![], + executed_block: Some(executed_block), + }); + + Availability::PendingBlobs(blob_ids) + } + }; + + Ok(availability) + } + + fn check_block_availability_or_cache( + &self, + cache: &mut GossipBlobCache, + executed_block: AvailabilityPendingExecutedBlock, + ) -> Result, AvailabilityCheckError> { + let AvailabilityPendingExecutedBlock { + block, + import_data, + payload_verification_outcome, + } = executed_block; + let kzg_commitments_len = block.kzg_commitments()?.len(); + let verified_commitments_len = cache.verified_blobs.len(); + if kzg_commitments_len == verified_commitments_len { + //TODO(sean) can we remove this clone + let blobs = cache.verified_blobs.clone(); + let available_block = self.make_available(block, blobs)?; + Ok(Availability::Available(Box::new( + AvailableExecutedBlock::new( + available_block, + import_data, + payload_verification_outcome, + ), + ))) + } else { + let mut missing_blobs = Vec::with_capacity(kzg_commitments_len); + for i in 0..kzg_commitments_len { + if cache.verified_blobs.get(i).is_none() { + missing_blobs.push(BlobIdentifier { + block_root: import_data.block_root, + index: i as u64, + }) + } + } + + let _ = cache + .executed_block + .insert(AvailabilityPendingExecutedBlock::new( + block, + import_data, + payload_verification_outcome, + )); + + Ok(Availability::PendingBlobs(missing_blobs)) + } + } + + /// Checks if a block is available, returns a MaybeAvailableBlock enum that may include the fully + /// available block. + pub fn check_availability( + &self, + block: BlockWrapper, + ) -> Result, AvailabilityCheckError> { + match block { + BlockWrapper::Block(block) => self.check_availability_without_blobs(block), + BlockWrapper::BlockAndBlobs(block, blob_list) => { + let kzg = self + .kzg + .as_ref() + .ok_or(AvailabilityCheckError::KzgNotInitialized)?; + let verified_blobs = verify_kzg_for_blob_list(blob_list, kzg)?; + + Ok(MaybeAvailableBlock::Available( + self.check_availability_with_blobs(block, verified_blobs)?, + )) + } + } + } + + /// Checks if a block is available, returning an error if the block is not immediately available. + /// Does not access the gossip cache. + pub fn try_check_availability( + &self, + block: BlockWrapper, + ) -> Result, AvailabilityCheckError> { + match block { + BlockWrapper::Block(block) => { + let blob_requirements = self.get_blob_requirements(&block)?; + let blobs = match blob_requirements { + BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs, + BlobRequirements::NotRequired => VerifiedBlobs::NotRequired, + BlobRequirements::PreEip4844 => VerifiedBlobs::PreEip4844, + BlobRequirements::Required => return Err(AvailabilityCheckError::MissingBlobs), + }; + Ok(AvailableBlock { block, blobs }) + } + BlockWrapper::BlockAndBlobs(_, _) => Err(AvailabilityCheckError::Pending), + } + } + + /// Verifies a block against a set of KZG verified blobs. Returns an AvailableBlock if block's + /// commitments are consistent with the provided verified blob commitments. + pub fn check_availability_with_blobs( + &self, + block: Arc>, + blobs: KzgVerifiedBlobList, + ) -> Result, AvailabilityCheckError> { + match self.check_availability_without_blobs(block)? { + MaybeAvailableBlock::Available(block) => Ok(block), + MaybeAvailableBlock::AvailabilityPending(pending_block) => { + self.make_available(pending_block, blobs) + } + } + } + + /// Verifies a block as much as possible, returning a MaybeAvailableBlock enum that may include + /// an AvailableBlock if no blobs are required. Otherwise this will return an AvailabilityPendingBlock. + pub fn check_availability_without_blobs( + &self, + block: Arc>, + ) -> Result, AvailabilityCheckError> { + let blob_requirements = self.get_blob_requirements(&block)?; + let blobs = match blob_requirements { + BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs, + BlobRequirements::NotRequired => VerifiedBlobs::NotRequired, + BlobRequirements::PreEip4844 => VerifiedBlobs::PreEip4844, + BlobRequirements::Required => { + return Ok(MaybeAvailableBlock::AvailabilityPending( + AvailabilityPendingBlock { block }, + )) + } + }; + Ok(MaybeAvailableBlock::Available(AvailableBlock { + block, + blobs, + })) + } + + /// Verifies an AvailabilityPendingBlock against a set of KZG verified blobs. + /// This does not check whether a block *should* have blobs, these checks should must have been + /// completed when producing the AvailabilityPendingBlock. + pub fn make_available( + &self, + block: AvailabilityPendingBlock, + blobs: KzgVerifiedBlobList, + ) -> Result, AvailabilityCheckError> { + let block_kzg_commitments = block.kzg_commitments()?; + if blobs.len() != block_kzg_commitments.len() { + return Err(AvailabilityCheckError::NumBlobsMismatch { + num_kzg_commitments: block_kzg_commitments.len(), + num_blobs: blobs.len(), + }); + } + + for (block_commitment, blob) in block_kzg_commitments.iter().zip(blobs.iter()) { + if *block_commitment != blob.kzg_commitment() { + return Err(AvailabilityCheckError::KzgCommitmentMismatch { + blob_index: blob.as_blob().index, + }); + } + } + + let blobs = VariableList::new(blobs.into_iter().map(|blob| blob.to_blob()).collect())?; + + Ok(AvailableBlock { + block: block.block, + blobs: VerifiedBlobs::Available(blobs), + }) + } + + /// Determines the blob requirements for a block. Answers the question: "Does this block require + /// blobs?". + fn get_blob_requirements( + &self, + block: &Arc>>, + ) -> Result { + let verified_blobs = if let (Ok(block_kzg_commitments), Ok(payload)) = ( + block.message().body().blob_kzg_commitments(), + block.message().body().execution_payload(), + ) { + if let Some(transactions) = payload.transactions() { + let verified = verify_kzg_commitments_against_transactions::( + transactions, + block_kzg_commitments, + ) + .map_err(|_| AvailabilityCheckError::TxKzgCommitmentMismatch)?; + if !verified { + return Err(AvailabilityCheckError::TxKzgCommitmentMismatch); + } + } + + if self.da_check_required(block.epoch()) { + if block_kzg_commitments.is_empty() { + BlobRequirements::EmptyBlobs + } else { + BlobRequirements::Required + } + } else { + BlobRequirements::NotRequired + } + } else { + BlobRequirements::PreEip4844 + }; + Ok(verified_blobs) + } + + /// The epoch at which we require a data availability check in block processing. + /// `None` if the `Eip4844` fork is disabled. + pub fn data_availability_boundary(&self) -> Option { + self.spec.eip4844_fork_epoch.and_then(|fork_epoch| { + self.slot_clock + .now() + .map(|slot| slot.epoch(T::slots_per_epoch())) + .map(|current_epoch| { + std::cmp::max( + fork_epoch, + current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS), + ) + }) + }) + } + + /// Returns true if the given epoch lies within the da boundary and false otherwise. + pub fn da_check_required(&self, block_epoch: Epoch) -> bool { + self.data_availability_boundary() + .map_or(false, |da_epoch| block_epoch >= da_epoch) + } +} + +pub enum BlobRequirements { + Required, + /// This block is from outside the data availability boundary so doesn't require + /// a data availability check. + NotRequired, + /// The block's `kzg_commitments` field is empty so it does not contain any blobs. + EmptyBlobs, + /// This is a block prior to the 4844 fork, so doesn't require any blobs + PreEip4844, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct AvailabilityPendingBlock { + block: Arc>, +} + +impl AvailabilityPendingBlock { + pub fn to_block(self) -> Arc> { + self.block + } + pub fn as_block(&self) -> &SignedBeaconBlock { + &self.block + } + pub fn block_cloned(&self) -> Arc> { + self.block.clone() + } + pub fn kzg_commitments(&self) -> Result<&KzgCommitments, AvailabilityCheckError> { + self.block + .message() + .body() + .blob_kzg_commitments() + .map_err(|_| AvailabilityCheckError::IncorrectFork) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct AvailableBlock { + block: Arc>, + blobs: VerifiedBlobs, +} + +impl AvailableBlock { + pub fn block(&self) -> &SignedBeaconBlock { + &self.block + } + + pub fn deconstruct(self) -> (Arc>, Option>) { + match self.blobs { + VerifiedBlobs::EmptyBlobs | VerifiedBlobs::NotRequired | VerifiedBlobs::PreEip4844 => { + (self.block, None) + } + VerifiedBlobs::Available(blobs) => (self.block, Some(blobs)), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum VerifiedBlobs { + /// These blobs are available. + Available(BlobSidecarList), + /// This block is from outside the data availability boundary so doesn't require + /// a data availability check. + NotRequired, + /// The block's `kzg_commitments` field is empty so it does not contain any blobs. + EmptyBlobs, + /// This is a block prior to the 4844 fork, so doesn't require any blobs + PreEip4844, +} + +impl AsBlock for AvailableBlock { + fn slot(&self) -> Slot { + self.block.slot() + } + + fn epoch(&self) -> Epoch { + self.block.epoch() + } + + fn parent_root(&self) -> Hash256 { + self.block.parent_root() + } + + fn state_root(&self) -> Hash256 { + self.block.state_root() + } + + fn signed_block_header(&self) -> SignedBeaconBlockHeader { + self.block.signed_block_header() + } + + fn message(&self) -> BeaconBlockRef { + self.block.message() + } + + fn as_block(&self) -> &SignedBeaconBlock { + &self.block + } + + fn block_cloned(&self) -> Arc> { + self.block.clone() + } + + fn canonical_root(&self) -> Hash256 { + self.block.canonical_root() + } + + fn into_block_wrapper(self) -> BlockWrapper { + let (block, blobs_opt) = self.deconstruct(); + if let Some(blobs) = blobs_opt { + BlockWrapper::BlockAndBlobs(block, blobs) + } else { + BlockWrapper::Block(block) + } + } +} diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index 5fe14c7e252..082c2242c84 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -1,4 +1,4 @@ -use crate::blob_verification::AvailableBlock; +use crate::data_availability_checker::AvailableBlock; use crate::{ attester_cache::{CommitteeLengths, Error}, metrics, @@ -6,6 +6,7 @@ use crate::{ use parking_lot::RwLock; use proto_array::Block as ProtoBlock; use std::sync::Arc; +use types::blob_sidecar::BlobSidecarList; use types::*; pub struct CacheItem { @@ -21,6 +22,7 @@ pub struct CacheItem { * Values used to make the block available. */ block: Arc>, + //TODO(sean) remove this and just use the da checker?' blobs: Option>, proto_block: ProtoBlock, } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index feafa7fd835..c76ba752a85 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -16,6 +16,7 @@ pub mod builder; pub mod canonical_head; pub mod capella_readiness; pub mod chain_config; +pub mod data_availability_checker; mod early_attester_cache; mod errors; pub mod eth1_chain; @@ -54,9 +55,10 @@ pub mod validator_monitor; pub mod validator_pubkey_cache; pub use self::beacon_chain::{ - AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult, - CountUnrealized, ForkChoiceError, OverrideForkchoiceUpdate, ProduceBlockVerification, - StateSkipConfig, WhenSlotSkipped, INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON, + AttestationProcessingOutcome, AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes, + BeaconStore, ChainSegmentResult, CountUnrealized, ForkChoiceError, OverrideForkchoiceUpdate, + ProduceBlockVerification, StateSkipConfig, WhenSlotSkipped, + INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON, INVALID_JUSTIFIED_PAYLOAD_SHUTDOWN_REASON, MAXIMUM_GOSSIP_CLOCK_DISPARITY, }; pub use self::beacon_snapshot::BeaconSnapshot; @@ -66,7 +68,7 @@ pub use self::historical_blocks::HistoricalBlockError; pub use attestation_verification::Error as AttestationError; pub use beacon_fork_choice_store::{BeaconForkChoiceStore, Error as ForkChoiceStoreError}; pub use block_verification::{ - get_block_root, BlockError, ExecutionPayloadError, GossipVerifiedBlock, + get_block_root, BlockError, ExecutedBlock, ExecutionPayloadError, GossipVerifiedBlock, }; pub use canonical_head::{CachedHead, CanonicalHead, CanonicalHeadRwLock}; pub use eth1_chain::{Eth1Chain, Eth1ChainBackend}; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index a784c674113..df2545adf12 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1680,7 +1680,8 @@ where NotifyExecutionLayer::Yes, ) .await? - .into(); + .try_into() + .unwrap(); self.chain.recompute_head_at_current_slot().await; Ok(block_hash) } @@ -1698,7 +1699,8 @@ where NotifyExecutionLayer::Yes, ) .await? - .into(); + .try_into() + .unwrap(); self.chain.recompute_head_at_current_slot().await; Ok(block_hash) } diff --git a/beacon_node/beacon_chain/tests/attestation_production.rs b/beacon_node/beacon_chain/tests/attestation_production.rs index 9cfff81c388..feea5c7218b 100644 --- a/beacon_node/beacon_chain/tests/attestation_production.rs +++ b/beacon_node/beacon_chain/tests/attestation_production.rs @@ -1,9 +1,7 @@ #![cfg(not(debug_assertions))] -use beacon_chain::{ - blob_verification::{BlockWrapper, IntoAvailableBlock}, - test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy}, -}; +use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy}; use beacon_chain::{StateSkipConfig, WhenSlotSkipped}; use lazy_static::lazy_static; use std::sync::Arc; @@ -135,6 +133,10 @@ async fn produces_attestations() { assert_eq!(data.target.root, target_root, "bad target root"); let block_wrapper: BlockWrapper = Arc::new(block.clone()).into(); + let available_block = chain + .data_availability_checker + .try_check_availability(block_wrapper) + .unwrap(); let early_attestation = { let proto_block = chain @@ -146,9 +148,7 @@ async fn produces_attestations() { .early_attester_cache .add_head_block( block_root, - block_wrapper - .into_available_block(block_root, chain) - .expect("should wrap into available block"), + available_block, proto_block, &state, &chain.spec, @@ -199,18 +199,19 @@ async fn early_attester_cache_old_request() { .get_block(&head.beacon_block_root) .unwrap(); - let block: BlockWrapper = head.beacon_block.clone().into(); + let block_wrapper: BlockWrapper = head.beacon_block.clone().into(); + let available_block = harness + .chain + .data_availability_checker + .try_check_availability(block_wrapper) + .unwrap(); - let chain = &harness.chain; harness .chain .early_attester_cache .add_head_block( head.beacon_block_root, - block - .clone() - .into_available_block(head.beacon_block_root, &chain) - .expect("should wrap into available block"), + available_block, head_proto_block, &head.beacon_state, &harness.chain.spec, diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 8de906afd97..88364d6ff5f 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -1,7 +1,8 @@ #![cfg(not(debug_assertions))] +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::{ - blob_verification::{AsBlock, BlockWrapper}, + blob_verification::AsBlock, test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType}, }; use beacon_chain::{BeaconSnapshot, BlockError, ChainSegmentResult, NotifyExecutionLayer}; diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index d0f81652bbd..dbb2ebfaea5 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -702,6 +702,8 @@ async fn invalidates_all_descendants() { NotifyExecutionLayer::Yes, ) .await + .unwrap() + .try_into() .unwrap(); rig.recompute_head().await; @@ -799,6 +801,8 @@ async fn switches_heads() { NotifyExecutionLayer::Yes, ) .await + .unwrap() + .try_into() .unwrap(); rig.recompute_head().await; diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index b4eabc8093f..f73cd0acad4 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -681,19 +681,20 @@ async fn run_skip_slot_test(skip_slots: u64) { Slot::new(0) ); - assert_eq!( - harness_b - .chain - .process_block( - harness_a.chain.head_snapshot().beacon_block_root, - harness_a.chain.head_snapshot().beacon_block.clone(), - CountUnrealized::True, - NotifyExecutionLayer::Yes, - ) - .await - .unwrap(), - harness_a.chain.head_snapshot().beacon_block_root - ); + let status = harness_b + .chain + .process_block( + harness_a.chain.head_snapshot().beacon_block_root, + harness_a.chain.head_snapshot().beacon_block.clone(), + CountUnrealized::True, + NotifyExecutionLayer::Yes, + ) + .await + .unwrap(); + + let root: Hash256 = status.try_into().unwrap(); + + assert_eq!(root, harness_a.chain.head_snapshot().beacon_block_root); harness_b.chain.recompute_head_at_current_slot().await; diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index e84e8dd8175..45ec4862605 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -2,9 +2,10 @@ use super::*; use serde::{Deserialize, Serialize}; use strum::EnumString; use superstruct::superstruct; -use types::blobs_sidecar::KzgCommitments; +use types::beacon_block_body::KzgCommitments; +use types::blob_sidecar::Blobs; use types::{ - Blobs, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, + EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, VariableList, Withdrawal, }; diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 2160c06e4dc..9704801d766 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -41,15 +41,16 @@ use tokio::{ }; use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; +use types::beacon_block_body::KzgCommitments; +use types::blob_sidecar::Blobs; use types::consts::eip4844::BLOB_TX_TYPE; use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlobTransaction}; use types::Withdrawals; +use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ - blobs_sidecar::{Blobs, KzgCommitments}, BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, ForkName, }; -use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Transaction, Uint256, diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index ef7affeb7f4..36675f74be2 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -218,10 +218,7 @@ impl BlockId { chain: &BeaconChain, ) -> Result, warp::Rejection> { let root = self.root(chain)?.0; - let Some(data_availability_boundary) = chain.data_availability_boundary() else { - return Err(warp_utils::reject::custom_not_found("Deneb fork disabled".into())); - }; - match chain.get_blobs(&root, data_availability_boundary) { + match chain.get_blobs(&root) { Ok(Some(blob_sidecar_list)) => Ok(blob_sidecar_list), Ok(None) => Err(warp_utils::reject::custom_not_found(format!( "No blobs with block root {} found in the store", diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index 9fbde0ce06a..d40fef1d908 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,7 +1,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProductionError}; -use eth2::types::BlockContents; +use eth2::types::{BeaconBlockAndBlobSidecars, BlockContents}; use std::sync::Arc; -use types::{AbstractExecPayload, BeaconBlock, BeaconBlockAndBlobSidecars, ForkName}; +use types::{AbstractExecPayload, BeaconBlock, ForkName}; type Error = warp::reject::Rejection; @@ -16,7 +16,7 @@ pub fn build_block_contents { let block_root = &block.canonical_root(); - if let Some(blob_sidecars) = chain.blob_cache.pop(block_root) { + if let Some(blob_sidecars) = chain.proposal_blob_cache.pop(block_root) { let block_and_blobs = BeaconBlockAndBlobSidecars { block, blob_sidecars, diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index b2d423633ba..c470686ad7f 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -1,10 +1,10 @@ use crate::metrics; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper, IntoAvailableBlock}; + +use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; -use beacon_chain::{ - BeaconChain, BeaconChainTypes, BlockError, CountUnrealized, NotifyExecutionLayer, -}; -use eth2::types::SignedBlockContents; +use beacon_chain::{AvailabilityProcessingStatus, NotifyExecutionLayer}; +use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, CountUnrealized}; +use eth2::types::{SignedBlockContents, VariableList}; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; use network::NetworkMessage; @@ -70,54 +70,51 @@ pub async fn publish_block( } SignedBeaconBlock::Eip4844(_) => { crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; - if let Some(blobs) = maybe_blobs { - for (blob_index, blob) in blobs.into_iter().enumerate() { + if let Some(signed_blobs) = maybe_blobs { + for (blob_index, blob) in signed_blobs.clone().into_iter().enumerate() { crate::publish_pubsub_message( network_tx, PubsubMessage::BlobSidecar(Box::new((blob_index as u64, blob))), )?; } + let blobs_vec = signed_blobs.into_iter().map(|blob| blob.message).collect(); + let blobs = VariableList::new(blobs_vec).map_err(|e| { + warp_utils::reject::custom_server_error(format!("Invalid blobs length: {e:?}")) + })?; + BlockWrapper::BlockAndBlobs(block, blobs) + } else { + block.into() } - block.into() - } - }; - - let available_block = match wrapped_block.into_available_block(block_root, &chain) { - Ok(available_block) => available_block, - Err(e) => { - let msg = format!("{:?}", e); - error!( - log, - "Invalid block provided to HTTP API"; - "reason" => &msg - ); - return Err(warp_utils::reject::broadcast_without_import(msg)); } }; + // Determine the delay after the start of the slot, register it with metrics. + let block_clone = wrapped_block.block_cloned(); + let slot = block_clone.message().slot(); + let proposer_index = block_clone.message().proposer_index(); match chain .process_block( block_root, - available_block.clone(), + wrapped_block, CountUnrealized::True, NotifyExecutionLayer::Yes, ) .await { - Ok(root) => { + Ok(AvailabilityProcessingStatus::Imported(root)) => { info!( log, "Valid block from HTTP API"; "block_delay" => ?delay, "root" => format!("{}", root), - "proposer_index" => available_block.message().proposer_index(), - "slot" => available_block.slot(), + "proposer_index" => proposer_index, + "slot" =>slot, ); // Notify the validator monitor. chain.validator_monitor.read().register_api_block( seen_timestamp, - available_block.message(), + block_clone.message(), root, &chain.slot_clock, ); @@ -133,7 +130,7 @@ pub async fn publish_block( late_block_logging( &chain, seen_timestamp, - available_block.message(), + block_clone.message(), root, "local", &log, @@ -142,12 +139,30 @@ pub async fn publish_block( Ok(()) } + Ok(AvailabilityProcessingStatus::PendingBlock(block_root)) => { + let msg = format!("Missing block with root {:?}", block_root); + error!( + log, + "Invalid block provided to HTTP API"; + "reason" => &msg + ); + Err(warp_utils::reject::broadcast_without_import(msg)) + } + Ok(AvailabilityProcessingStatus::PendingBlobs(blob_ids)) => { + let msg = format!("Missing blobs {:?}", blob_ids); + error!( + log, + "Invalid block provided to HTTP API"; + "reason" => &msg + ); + Err(warp_utils::reject::broadcast_without_import(msg)) + } Err(BlockError::BlockIsAlreadyKnown) => { info!( log, "Block from HTTP API already known"; "block" => ?block_root, - "slot" => available_block.slot(), + "slot" => slot, ); Ok(()) } diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index 6264d69333a..bf67c943598 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -20,10 +20,9 @@ use tokio_util::{ codec::Framed, compat::{Compat, FuturesAsyncReadCompatExt}, }; -use types::BlobsSidecar; use types::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockMerge, - EmptyBlock, EthSpec, ForkContext, ForkName, Hash256, MainnetEthSpec, Signature, + BlobSidecar, EmptyBlock, EthSpec, ForkContext, ForkName, Hash256, MainnetEthSpec, Signature, SignedBeaconBlock, }; @@ -115,12 +114,12 @@ lazy_static! { .as_ssz_bytes() .len(); - pub static ref BLOBS_SIDECAR_MIN: usize = BlobsSidecar::::empty().as_ssz_bytes().len(); - pub static ref BLOBS_SIDECAR_MAX: usize = BlobsSidecar::::max_size(); + pub static ref BLOB_SIDECAR_MIN: usize = BlobSidecar::::empty().as_ssz_bytes().len(); + pub static ref BLOB_SIDECAR_MAX: usize = BlobSidecar::::max_size(); //FIXME(sean) these are underestimates - pub static ref SIGNED_BLOCK_AND_BLOBS_MIN: usize = *BLOBS_SIDECAR_MIN + *SIGNED_BEACON_BLOCK_BASE_MIN; - pub static ref SIGNED_BLOCK_AND_BLOBS_MAX: usize =*BLOBS_SIDECAR_MAX + *SIGNED_BEACON_BLOCK_EIP4844_MAX; + pub static ref SIGNED_BLOCK_AND_BLOBS_MIN: usize = *BLOB_SIDECAR_MIN + *SIGNED_BEACON_BLOCK_BASE_MIN; + pub static ref SIGNED_BLOCK_AND_BLOBS_MAX: usize =*BLOB_SIDECAR_MAX + *SIGNED_BEACON_BLOCK_EIP4844_MAX; } /// The maximum bytes that can be sent across the RPC pre-merge. @@ -385,7 +384,7 @@ impl ProtocolId { Protocol::Goodbye => RpcLimits::new(0, 0), // Goodbye request has no response Protocol::BlocksByRange => rpc_block_limits_by_fork(fork_context.current_fork()), Protocol::BlocksByRoot => rpc_block_limits_by_fork(fork_context.current_fork()), - Protocol::BlobsByRange => RpcLimits::new(*BLOBS_SIDECAR_MIN, *BLOBS_SIDECAR_MAX), + Protocol::BlobsByRange => RpcLimits::new(*BLOB_SIDECAR_MIN, *BLOB_SIDECAR_MAX), Protocol::BlobsByRoot => { // TODO: wrong too RpcLimits::new(*SIGNED_BLOCK_AND_BLOBS_MIN, *SIGNED_BLOCK_AND_BLOBS_MAX) diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index d068a20079b..ea415c00558 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -41,7 +41,7 @@ num_cpus = "1.13.0" lru_cache = { path = "../../common/lru_cache" } if-addrs = "0.6.4" strum = "0.24.0" -tokio-util = { version = "0.6.3", features = ["time"] } +tokio-util = { version = "0.7.7", features = ["time"] } derivative = "2.2.0" delay_map = "0.3.0" ethereum-types = { version = "0.14.1", optional = true } diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/network/src/beacon_processor/mod.rs index 410389b1195..f58f2813515 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/network/src/beacon_processor/mod.rs @@ -449,7 +449,7 @@ impl WorkEvent { peer_id: PeerId, peer_client: Client, blob_index: u64, - signed_blob: Arc>, + signed_blob: SignedBlobSidecar, seen_timestamp: Duration, ) -> Self { Self { @@ -459,7 +459,7 @@ impl WorkEvent { peer_id, peer_client, blob_index, - signed_blob, + signed_blob: Box::new(signed_blob), seen_timestamp, }, } @@ -729,7 +729,7 @@ impl WorkEvent { impl std::convert::From> for WorkEvent { fn from(ready_work: ReadyWork) -> Self { match ready_work { - ReadyWork::Block(QueuedGossipBlock { + ReadyWork::GossipBlock(QueuedGossipBlock { peer_id, block, seen_timestamp, @@ -864,7 +864,7 @@ pub enum Work { peer_id: PeerId, peer_client: Client, blob_index: u64, - signed_blob: Arc>, + signed_blob: Box>, seen_timestamp: Duration, }, DelayedImportBlock { @@ -1759,7 +1759,7 @@ impl BeaconProcessor { peer_id, peer_client, blob_index, - signed_blob, + *signed_blob, seen_timestamp, ) .await diff --git a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs b/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs index 0e4a08f5d6c..4a565307991 100644 --- a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs +++ b/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs @@ -13,14 +13,15 @@ use super::MAX_SCHEDULED_WORK_QUEUE_LEN; use crate::metrics; use crate::sync::manager::BlockProcessType; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::{BeaconChainTypes, GossipVerifiedBlock, MAXIMUM_GOSSIP_CLOCK_DISPARITY}; use fnv::FnvHashMap; use futures::task::Poll; use futures::{Stream, StreamExt}; use lighthouse_network::{MessageId, PeerId}; use logging::TimeLatch; -use slog::{crit, debug, error, trace, warn, Logger}; +use slog::{debug, error, trace, warn, Logger}; use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; use std::pin::Pin; @@ -28,7 +29,6 @@ use std::task::Context; use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, Receiver, Sender}; -use tokio::time::error::Error as TimeError; use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey}; use types::{ Attestation, EthSpec, Hash256, LightClientOptimisticUpdate, SignedAggregateAndProof, SubnetId, @@ -87,7 +87,7 @@ pub enum ReprocessQueueMessage { /// Events sent by the scheduler once they are ready for re-processing. pub enum ReadyWork { - Block(QueuedGossipBlock), + GossipBlock(QueuedGossipBlock), RpcBlock(QueuedRpcBlock), Unaggregate(QueuedUnaggregate), Aggregate(QueuedAggregate), @@ -154,8 +154,6 @@ enum InboundEvent { ReadyAttestation(QueuedAttestationId), /// A light client update that is ready for re-processing. ReadyLightClientUpdate(QueuedLightClientUpdateId), - /// A `DelayQueue` returned an error. - DelayQueueError(TimeError, &'static str), /// A message sent to the `ReprocessQueue` Msg(ReprocessQueueMessage), } @@ -233,54 +231,42 @@ impl Stream for ReprocessQueue { // The sequential nature of blockchains means it is generally better to try and import all // existing blocks before new ones. match self.gossip_block_delay_queue.poll_expired(cx) { - Poll::Ready(Some(Ok(queued_block))) => { + Poll::Ready(Some(queued_block)) => { return Poll::Ready(Some(InboundEvent::ReadyGossipBlock( queued_block.into_inner(), ))); } - Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "gossip_block_queue"))); - } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), } match self.rpc_block_delay_queue.poll_expired(cx) { - Poll::Ready(Some(Ok(queued_block))) => { + Poll::Ready(Some(queued_block)) => { return Poll::Ready(Some(InboundEvent::ReadyRpcBlock(queued_block.into_inner()))); } - Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "rpc_block_queue"))); - } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), } match self.attestations_delay_queue.poll_expired(cx) { - Poll::Ready(Some(Ok(attestation_id))) => { + Poll::Ready(Some(attestation_id)) => { return Poll::Ready(Some(InboundEvent::ReadyAttestation( attestation_id.into_inner(), ))); } - Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "attestations_queue"))); - } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), } match self.lc_updates_delay_queue.poll_expired(cx) { - Poll::Ready(Some(Ok(lc_id))) => { + Poll::Ready(Some(lc_id)) => { return Poll::Ready(Some(InboundEvent::ReadyLightClientUpdate( lc_id.into_inner(), ))); } - Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "lc_updates_queue"))); - } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), @@ -400,7 +386,7 @@ impl ReprocessQueue { if block_slot <= now && self .ready_work_tx - .try_send(ReadyWork::Block(early_block)) + .try_send(ReadyWork::GossipBlock(early_block)) .is_err() { error!( @@ -694,7 +680,7 @@ impl ReprocessQueue { if self .ready_work_tx - .try_send(ReadyWork::Block(ready_block)) + .try_send(ReadyWork::GossipBlock(ready_block)) .is_err() { error!( @@ -703,14 +689,7 @@ impl ReprocessQueue { ); } } - InboundEvent::DelayQueueError(e, queue_name) => { - crit!( - log, - "Failed to poll queue"; - "queue" => queue_name, - "e" => ?e - ) - } + InboundEvent::ReadyAttestation(queued_id) => { metrics::inc_counter( &metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_ATTESTATIONS, diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index e5b4bdf86a6..69e167b4d03 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -1,6 +1,6 @@ use crate::{metrics, service::NetworkMessage, sync::SyncMessage}; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::{AsBlock, BlockWrapper, GossipVerifiedBlob}; use beacon_chain::store::Error; use beacon_chain::{ attestation_verification::{self, Error as AttnError, VerifiedAttestation}, @@ -9,15 +9,14 @@ use beacon_chain::{ observed_operations::ObservationOutcome, sync_committee_verification::{self, Error as SyncCommitteeError}, validator_monitor::get_block_delay_ms, - BeaconChainError, BeaconChainTypes, BlockError, CountUnrealized, ForkChoiceError, - GossipVerifiedBlock, NotifyExecutionLayer, + AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, CountUnrealized, + ForkChoiceError, GossipVerifiedBlock, NotifyExecutionLayer, }; use lighthouse_network::{Client, MessageAcceptance, MessageId, PeerAction, PeerId, ReportSource}; use operation_pool::ReceivedPreCapella; use slog::{crit, debug, error, info, trace, warn}; use slot_clock::SlotClock; use ssz::Encode; -use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tokio::sync::mpsc; @@ -654,19 +653,57 @@ impl Worker { self, _message_id: MessageId, peer_id: PeerId, - peer_client: Client, + _peer_client: Client, blob_index: u64, - signed_blob: Arc>, + signed_blob: SignedBlobSidecar, _seen_duration: Duration, ) { - // TODO: gossip verification - crit!(self.log, "UNIMPLEMENTED gossip blob verification"; - "peer_id" => %peer_id, - "client" => %peer_client, - "blob_topic" => blob_index, - "blob_index" => signed_blob.message.index, - "blob_slot" => signed_blob.message.slot - ); + match self + .chain + .verify_blob_sidecar_for_gossip(signed_blob, blob_index) + { + Ok(gossip_verified_blob) => { + self.process_gossip_verified_blob(peer_id, gossip_verified_blob, _seen_duration) + .await + } + Err(_) => { + // TODO(pawan): handle all blob errors for peer scoring + todo!() + } + } + } + + pub async fn process_gossip_verified_blob( + self, + peer_id: PeerId, + verified_blob: GossipVerifiedBlob, + // This value is not used presently, but it might come in handy for debugging. + _seen_duration: Duration, + ) { + // TODO + match self + .chain + .process_blob(verified_blob, CountUnrealized::True) + .await + { + Ok(AvailabilityProcessingStatus::Imported(_hash)) => { + todo!() + // add to metrics + // logging + } + Ok(AvailabilityProcessingStatus::PendingBlobs(pending_blobs)) => self + .send_sync_message(SyncMessage::UnknownBlobHash { + peer_id, + pending_blobs, + }), + Ok(AvailabilityProcessingStatus::PendingBlock(block_hash)) => { + self.send_sync_message(SyncMessage::UnknownBlockHash(peer_id, block_hash)); + } + Err(_err) => { + // handle errors + todo!() + } + } } /// Process the beacon block received from the gossip network and: @@ -802,6 +839,9 @@ impl Worker { verified_block } + Err(BlockError::AvailabilityCheck(_err)) => { + todo!() + } Err(BlockError::ParentUnknown(block)) => { debug!( self.log, @@ -984,7 +1024,7 @@ impl Worker { ) .await { - Ok(block_root) => { + Ok(AvailabilityProcessingStatus::Imported(block_root)) => { metrics::inc_counter(&metrics::BEACON_PROCESSOR_GOSSIP_BLOCK_IMPORTED_TOTAL); if reprocess_tx @@ -1011,6 +1051,24 @@ impl Worker { self.chain.recompute_head_at_current_slot().await; } + Ok(AvailabilityProcessingStatus::PendingBlock(block_root)) => { + // This error variant doesn't make any sense in this context + crit!( + self.log, + "Internal error. Cannot get AvailabilityProcessingStatus::PendingBlock on processing block"; + "block_root" => %block_root + ); + } + Ok(AvailabilityProcessingStatus::PendingBlobs(pending_blobs)) => { + // make rpc request for blob + self.send_sync_message(SyncMessage::UnknownBlobHash { + peer_id, + pending_blobs, + }); + } + Err(BlockError::AvailabilityCheck(_)) => { + todo!() + } Err(BlockError::ParentUnknown(block)) => { // Inform the sync manager to find parents for this block // This should not occur. It should be checked by `should_forward_block` diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 45c31b77ed1..ecb2eb424ba 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use crate::beacon_processor::{worker::FUTURE_SLOT_TOLERANCE, SendOnDrop}; use crate::service::NetworkMessage; use crate::status::ToStatusMessage; @@ -15,7 +13,6 @@ use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo use slog::{debug, error, warn}; use slot_clock::SlotClock; use std::collections::{hash_map::Entry, HashMap}; -use std::sync::Arc; use task_executor::TaskExecutor; use tokio_stream::StreamExt; use types::blob_sidecar::BlobIdentifier; @@ -840,7 +837,7 @@ impl Worker { let mut send_response = true; for root in block_roots { - match self.chain.get_blobs(&root, data_availability_boundary) { + match self.chain.get_blobs(&root) { Ok(Some(blob_sidecar_list)) => { for blob_sidecar in blob_sidecar_list.iter() { blobs_sent += 1; diff --git a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs b/beacon_node/network/src/beacon_processor/worker/sync_methods.rs index 8b7e0f2cbe6..09e98cb183c 100644 --- a/beacon_node/network/src/beacon_processor/worker/sync_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/sync_methods.rs @@ -7,8 +7,9 @@ use crate::beacon_processor::DuplicateCache; use crate::metrics; use crate::sync::manager::{BlockProcessType, SyncMessage}; use crate::sync::{BatchProcessResult, ChainId}; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper, IntoAvailableBlock}; -use beacon_chain::CountUnrealized; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; +use beacon_chain::{AvailabilityProcessingStatus, CountUnrealized}; use beacon_chain::{ BeaconChainError, BeaconChainTypes, BlockError, ChainSegmentResult, HistoricalBlockError, NotifyExecutionLayer, @@ -86,28 +87,22 @@ impl Worker { }; let slot = block.slot(); let parent_root = block.message().parent_root(); - let available_block = block - .into_available_block(block_root, &self.chain) - .map_err(BlockError::BlobValidation); - let result = match available_block { - Ok(block) => { - self.chain - .process_block( - block_root, - block, - CountUnrealized::True, - NotifyExecutionLayer::Yes, - ) - .await - } - Err(e) => Err(e), - }; + let result = self + .chain + .process_block( + block_root, + block, + CountUnrealized::True, + NotifyExecutionLayer::Yes, + ) + .await; metrics::inc_counter(&metrics::BEACON_PROCESSOR_RPC_BLOCK_IMPORTED_TOTAL); // RPC block imported, regardless of process type - if let &Ok(hash) = &result { + //TODO(sean) handle pending availability variants + if let &Ok(AvailabilityProcessingStatus::Imported(hash)) = &result { info!(self.log, "New RPC block received"; "slot" => slot, "hash" => %hash); // Trigger processing for work referencing this block. diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 5e2cd6181aa..fed799988be 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -280,7 +280,6 @@ impl Router { PubsubMessage::BlobSidecar(data) => { let (blob_index, signed_blob) = *data; let peer_client = self.network_globals.client(&peer_id); - let signed_blob = Arc::new(signed_blob); self.send_beacon_processor_work(BeaconWorkEvent::gossip_signed_blob_sidecar( message_id, peer_id, diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 690d56644cd..77e659a2682 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -2,7 +2,8 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; use std::time::Duration; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::{BeaconChainTypes, BlockError}; use fnv::FnvHashMap; use lighthouse_network::rpc::{RPCError, RPCResponseErrorCode}; diff --git a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs index b6de52d7059..f066191c00a 100644 --- a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs @@ -1,5 +1,6 @@ use super::RootBlockTuple; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::BeaconChainTypes; use lighthouse_network::PeerId; use store::Hash256; diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 02482d6192d..60911dbb395 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -1,5 +1,6 @@ use super::RootBlockTuple; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::get_block_root; use lighthouse_network::{rpc::BlocksByRootRequest, PeerId}; use rand::seq::IteratorRandom; diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 43921b585a4..482bfab7083 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -42,7 +42,8 @@ use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent as BeaconWorkEven use crate::service::NetworkMessage; use crate::status::ToStatusMessage; use crate::sync::range_sync::ByRangeRequestType; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; +use beacon_chain::blob_verification::AsBlock; +use beacon_chain::blob_verification::BlockWrapper; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, EngineState}; use futures::StreamExt; use lighthouse_network::rpc::methods::MAX_REQUEST_BLOCKS; @@ -56,6 +57,7 @@ use std::ops::Sub; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; +use types::blob_sidecar::BlobIdentifier; use types::{BlobSidecar, EthSpec, Hash256, SignedBeaconBlock, Slot}; /// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync @@ -119,6 +121,13 @@ pub enum SyncMessage { /// manager to attempt to find the block matching the unknown hash. UnknownBlockHash(PeerId, Hash256), + /// A peer has sent us a block that we haven't received all the blobs for. This triggers + /// the manager to attempt to find the pending blobs for the given block root. + UnknownBlobHash { + peer_id: PeerId, + pending_blobs: Vec, + }, + /// A peer has disconnected. Disconnect(PeerId), @@ -598,6 +607,9 @@ impl SyncManager { .search_block(block_hash, peer_id, &mut self.network); } } + SyncMessage::UnknownBlobHash { .. } => { + unimplemented!() + } SyncMessage::Disconnect(peer_id) => { self.peer_disconnect(&peer_id); } diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 2c80c2c1ac7..fc902d866f5 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -38,6 +38,7 @@ use std::marker::PhantomData; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; +use types::blob_sidecar::BlobSidecarList; use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; use types::*; @@ -547,7 +548,7 @@ impl, Cold: ItemStore> HotColdDB /// Check if the blobs sidecar for a block exists on disk. pub fn blobs_sidecar_exists(&self, block_root: &Hash256) -> Result { - self.get_item::>(block_root) + self.get_item::>(block_root) .map(|blobs| blobs.is_some()) } diff --git a/beacon_node/store/src/impls/execution_payload.rs b/beacon_node/store/src/impls/execution_payload.rs index 01a2dba0b0a..f4e6c1ea661 100644 --- a/beacon_node/store/src/impls/execution_payload.rs +++ b/beacon_node/store/src/impls/execution_payload.rs @@ -1,7 +1,7 @@ use crate::{DBColumn, Error, StoreItem}; use ssz::{Decode, Encode}; use types::{ - BlobsSidecar, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, + BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, }; @@ -25,7 +25,7 @@ macro_rules! impl_store_item { impl_store_item!(ExecutionPayloadMerge); impl_store_item!(ExecutionPayloadCapella); impl_store_item!(ExecutionPayloadEip4844); -impl_store_item!(BlobsSidecar); +impl_store_item!(BlobSidecarList); /// This fork-agnostic implementation should be only used for writing. /// diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 29fded5fa6d..179b099de35 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -43,6 +43,7 @@ pub use metrics::scrape_for_metrics; use parking_lot::MutexGuard; use std::sync::Arc; use strum::{EnumString, IntoStaticStr}; +use types::blob_sidecar::BlobSidecarList; pub use types::*; pub type ColumnIter<'a> = Box), Error>> + 'a>; diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index bc27ddb4743..543b3fda668 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1396,3 +1396,32 @@ pub struct SignedBeaconBlockAndBlobSidecars, pub signed_blob_sidecars: SignedBlobSidecarList, } + +#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] +pub struct BeaconBlockAndBlobSidecars> { + pub block: BeaconBlock, + pub blob_sidecars: BlobSidecarList, +} + +impl> ForkVersionDeserialize + for BeaconBlockAndBlobSidecars +{ + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::value::Value, + fork_name: ForkName, + ) -> Result { + #[derive(Deserialize)] + #[serde(bound = "T: EthSpec")] + struct Helper { + block: serde_json::Value, + blob_sidecars: BlobSidecarList, + } + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + + Ok(Self { + block: BeaconBlock::deserialize_by_fork::<'de, D>(helper.block, fork_name)?, + blob_sidecars: helper.blob_sidecars, + }) + } +} diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index 4c8966f92cc..37bd5fe446d 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -7,7 +7,7 @@ use types::{ ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, SignedBeaconBlock, Slot, }; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ConsensusContext { /// Slot to act as an identifier/safeguard slot: Slot, diff --git a/consensus/types/src/beacon_block_and_blob_sidecars.rs b/consensus/types/src/beacon_block_and_blob_sidecars.rs deleted file mode 100644 index 78e70419614..00000000000 --- a/consensus/types/src/beacon_block_and_blob_sidecars.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::{ - AbstractExecPayload, BeaconBlock, BlobSidecarList, EthSpec, ForkName, ForkVersionDeserialize, -}; -use derivative::Derivative; -use serde_derive::{Deserialize, Serialize}; -use ssz_derive::Encode; -use tree_hash_derive::TreeHash; - -#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] -#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] -pub struct BeaconBlockAndBlobSidecars> { - pub block: BeaconBlock, - pub blob_sidecars: BlobSidecarList, -} - -impl> ForkVersionDeserialize - for BeaconBlockAndBlobSidecars -{ - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { - #[derive(Deserialize)] - #[serde(bound = "T: EthSpec")] - struct Helper { - block: serde_json::Value, - blob_sidecars: BlobSidecarList, - } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; - - Ok(Self { - block: BeaconBlock::deserialize_by_fork::<'de, D>(helper.block, fork_name)?, - blob_sidecars: helper.blob_sidecars, - }) - } -} diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index c7173965224..e49f633459f 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -1,5 +1,5 @@ +use crate::test_utils::TestRandom; use crate::*; -use crate::{blobs_sidecar::KzgCommitments, test_utils::TestRandom}; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -9,6 +9,8 @@ use superstruct::superstruct; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; +pub type KzgCommitments = VariableList::MaxBlobsPerBlock>; + /// The body of a `BeaconChain` block, containing operations. /// /// This *superstruct* abstracts over the hard-fork. diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 29eaadc5842..12f8afd9c73 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -11,7 +11,7 @@ use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; /// Container of the data that identifies an individual blob. -#[derive(Encode, Decode, Clone, Debug, PartialEq)] +#[derive(Encode, Decode, Clone, Debug, PartialEq, Eq, Hash)] pub struct BlobIdentifier { pub block_root: Hash256, pub index: u64, @@ -35,7 +35,6 @@ pub struct BlobIdentifier { #[derivative(PartialEq, Hash(bound = "T: EthSpec"))] pub struct BlobSidecar { pub block_root: Hash256, - // TODO: fix the type, should fit in u8 as well #[serde(with = "eth2_serde_utils::quoted_u64")] pub index: u64, pub slot: Slot, @@ -49,10 +48,18 @@ pub struct BlobSidecar { } pub type BlobSidecarList = VariableList>, ::MaxBlobsPerBlock>; +pub type Blobs = VariableList, ::MaxExtraDataBytes>; impl SignedRoot for BlobSidecar {} impl BlobSidecar { + pub fn id(&self) -> BlobIdentifier { + BlobIdentifier { + block_root: self.block_root, + index: self.index, + } + } + pub fn empty() -> Self { Self::default() } diff --git a/consensus/types/src/blobs_sidecar.rs b/consensus/types/src/blobs_sidecar.rs deleted file mode 100644 index e2560fb30bf..00000000000 --- a/consensus/types/src/blobs_sidecar.rs +++ /dev/null @@ -1,62 +0,0 @@ -use crate::test_utils::TestRandom; -use crate::{Blob, EthSpec, Hash256, KzgCommitment, SignedRoot, Slot}; -use derivative::Derivative; -use kzg::KzgProof; -use serde_derive::{Deserialize, Serialize}; -use ssz::Encode; -use ssz_derive::{Decode, Encode}; -use ssz_types::VariableList; -use test_random_derive::TestRandom; -use tree_hash_derive::TreeHash; - -pub type KzgCommitments = VariableList::MaxBlobsPerBlock>; -pub type Blobs = VariableList, ::MaxBlobsPerBlock>; - -#[derive( - Debug, - Clone, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - Default, - TestRandom, - Derivative, - arbitrary::Arbitrary, -)] -#[serde(bound = "T: EthSpec")] -#[arbitrary(bound = "T: EthSpec")] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] -pub struct BlobsSidecar { - pub beacon_block_root: Hash256, - pub beacon_block_slot: Slot, - #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] - pub blobs: Blobs, - pub kzg_aggregated_proof: KzgProof, -} - -impl SignedRoot for BlobsSidecar {} - -impl BlobsSidecar { - pub fn empty() -> Self { - Self::default() - } - - pub fn empty_from_parts(beacon_block_root: Hash256, beacon_block_slot: Slot) -> Self { - Self { - beacon_block_root, - beacon_block_slot, - blobs: VariableList::empty(), - kzg_aggregated_proof: KzgProof::empty(), - } - } - - #[allow(clippy::integer_arithmetic)] - pub fn max_size() -> usize { - // Fixed part - Self::empty().as_ssz_bytes().len() - // Max size of variable length `blobs` field - + (T::max_blobs_per_block() * as Encode>::ssz_fixed_len()) - } -} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 6a86e773a1b..14f3ff3560b 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -97,11 +97,8 @@ pub mod slot_data; #[cfg(feature = "sqlite")] pub mod sqlite; -pub mod beacon_block_and_blob_sidecars; pub mod blob_sidecar; -pub mod blobs_sidecar; pub mod signed_blob; -pub mod signed_block_and_blobs; pub mod transaction; use ethereum_types::{H160, H256}; @@ -115,7 +112,6 @@ pub use crate::beacon_block::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockEip4844, BeaconBlockMerge, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, EmptyBlock, }; -pub use crate::beacon_block_and_blob_sidecars::BeaconBlockAndBlobSidecars; pub use crate::beacon_block_body::{ BeaconBlockBody, BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyCapella, BeaconBlockBodyEip4844, BeaconBlockBodyMerge, BeaconBlockBodyRef, BeaconBlockBodyRefMut, @@ -124,7 +120,6 @@ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList}; -pub use crate::blobs_sidecar::{Blobs, BlobsSidecar}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; @@ -183,9 +178,6 @@ pub use crate::signed_beacon_block::{ }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_blob::*; -pub use crate::signed_block_and_blobs::{ - SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockAndBlobsSidecarDecode, -}; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; pub use crate::signed_contribution_and_proof::SignedContributionAndProof; pub use crate::signed_voluntary_exit::SignedVoluntaryExit; diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index ae59690bf2d..301cfd5f878 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -35,14 +35,6 @@ impl From for Hash256 { } } -#[derive(Debug)] -pub enum BlobReconstructionError { - /// No blobs for the specified block where we would expect blobs. - UnavailableBlobs, - /// Blobs provided for a pre-Eip4844 fork. - InconsistentFork, -} - /// A `BeaconBlock` and a signature from its proposer. #[superstruct( variants(Base, Altair, Merge, Capella, Eip4844), @@ -250,28 +242,6 @@ impl> SignedBeaconBlock pub fn canonical_root(&self) -> Hash256 { self.message().tree_hash_root() } - - /// Reconstructs an empty `BlobsSidecar`, using the given block root if provided, else calculates it. - /// If this block has kzg commitments, an error will be returned. If this block is from prior to the - /// Eip4844 fork, this will error. - pub fn reconstruct_empty_blobs( - &self, - block_root_opt: Option, - ) -> Result, BlobReconstructionError> { - let kzg_commitments = self - .message() - .body() - .blob_kzg_commitments() - .map_err(|_| BlobReconstructionError::InconsistentFork)?; - if kzg_commitments.is_empty() { - Ok(BlobsSidecar::empty_from_parts( - block_root_opt.unwrap_or_else(|| self.canonical_root()), - self.slot(), - )) - } else { - Err(BlobReconstructionError::UnavailableBlobs) - } - } } // We can convert pre-Bellatrix blocks without payloads into blocks with payloads. diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index 4eb28794ed5..aaab02ca783 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -1,10 +1,15 @@ -use crate::{test_utils::TestRandom, BlobSidecar, EthSpec, Signature}; +use crate::{ + test_utils::TestRandom, BlobSidecar, ChainSpec, Domain, EthSpec, Fork, Hash256, Signature, + SignedRoot, SigningData, +}; +use bls::PublicKey; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; use std::sync::Arc; use test_random_derive::TestRandom; +use tree_hash::TreeHash; use tree_hash_derive::TreeHash; #[derive( @@ -30,3 +35,37 @@ pub struct SignedBlobSidecar { pub type SignedBlobSidecarList = VariableList, ::MaxBlobsPerBlock>; + +impl SignedBlobSidecar { + /// Verify `self.signature`. + /// + /// If the root of `block.message` is already known it can be passed in via `object_root_opt`. + /// Otherwise, it will be computed locally. + pub fn verify_signature( + &self, + object_root_opt: Option, + pubkey: &PublicKey, + fork: &Fork, + genesis_validators_root: Hash256, + spec: &ChainSpec, + ) -> bool { + let domain = spec.get_domain( + self.message.slot.epoch(T::slots_per_epoch()), + Domain::BlobSidecar, + fork, + genesis_validators_root, + ); + + let message = if let Some(object_root) = object_root_opt { + SigningData { + object_root, + domain, + } + .tree_hash_root() + } else { + self.message.signing_root(domain) + }; + + self.signature.verify(pubkey, message) + } +} diff --git a/consensus/types/src/signed_block_and_blobs.rs b/consensus/types/src/signed_block_and_blobs.rs deleted file mode 100644 index a3bd34475d7..00000000000 --- a/consensus/types/src/signed_block_and_blobs.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::{BlobsSidecar, EthSpec, SignedBeaconBlock, SignedBeaconBlockEip4844}; -use derivative::Derivative; -use serde_derive::{Deserialize, Serialize}; -use ssz::{Decode, DecodeError}; -use ssz_derive::{Decode, Encode}; -use std::sync::Arc; -use tree_hash_derive::TreeHash; - -#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, PartialEq)] -#[serde(bound = "T: EthSpec")] -pub struct SignedBeaconBlockAndBlobsSidecarDecode { - pub beacon_block: SignedBeaconBlockEip4844, - pub blobs_sidecar: BlobsSidecar, -} - -// TODO: will be removed once we decouple blobs in Gossip -#[derive(Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative)] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] -pub struct SignedBeaconBlockAndBlobsSidecar { - pub beacon_block: Arc>, - pub blobs_sidecar: Arc>, -} - -impl SignedBeaconBlockAndBlobsSidecar { - pub fn from_ssz_bytes(bytes: &[u8]) -> Result { - let SignedBeaconBlockAndBlobsSidecarDecode { - beacon_block, - blobs_sidecar, - } = SignedBeaconBlockAndBlobsSidecarDecode::from_ssz_bytes(bytes)?; - Ok(SignedBeaconBlockAndBlobsSidecar { - beacon_block: Arc::new(SignedBeaconBlock::Eip4844(beacon_block)), - blobs_sidecar: Arc::new(blobs_sidecar), - }) - } -} diff --git a/lcli/src/parse_ssz.rs b/lcli/src/parse_ssz.rs index 0e87e330b13..70f34e09e3c 100644 --- a/lcli/src/parse_ssz.rs +++ b/lcli/src/parse_ssz.rs @@ -63,7 +63,7 @@ pub fn run_parse_ssz(matches: &ArgMatches) -> Result<(), String> { "state_merge" => decode_and_print::>(&bytes, format)?, "state_capella" => decode_and_print::>(&bytes, format)?, "state_eip4844" => decode_and_print::>(&bytes, format)?, - "blobs_sidecar" => decode_and_print::>(&bytes, format)?, + "blob_sidecar" => decode_and_print::>(&bytes, format)?, other => return Err(format!("Unknown type: {}", other)), }; diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 19b3535fbfa..d94dfef4854 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -50,7 +50,7 @@ type_name_generic!(BeaconBlockBodyCapella, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyEip4844, "BeaconBlockBody"); type_name!(BeaconBlockHeader); type_name_generic!(BeaconState); -type_name_generic!(BlobsSidecar); +type_name_generic!(BlobSidecar); type_name!(Checkpoint); type_name_generic!(ContributionAndProof); type_name!(Deposit); @@ -88,9 +88,5 @@ type_name!(Validator); type_name!(VoluntaryExit); type_name!(Withdrawal); type_name!(BlsToExecutionChange, "BLSToExecutionChange"); -type_name_generic!( - SignedBeaconBlockAndBlobsSidecarDecode, - "SignedBeaconBlockAndBlobsSidecar" -); type_name!(SignedBlsToExecutionChange, "SignedBLSToExecutionChange"); type_name!(HistoricalSummary); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index fe2c5f796fc..d5b8dc80a61 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -216,7 +216,6 @@ macro_rules! ssz_static_test_no_run { mod ssz_static { use ef_tests::{Handler, SszStaticHandler, SszStaticTHCHandler, SszStaticWithSpecHandler}; use types::historical_summary::HistoricalSummary; - use types::signed_block_and_blobs::SignedBeaconBlockAndBlobsSidecarDecode; use types::*; ssz_static_test!(aggregate_and_proof, AggregateAndProof<_>); @@ -378,12 +377,6 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::eip4844_only().run(); } - #[test] - fn signed_blobs_sidecar() { - SszStaticHandler::, MinimalEthSpec>::eip4844_only().run(); - SszStaticHandler::, MainnetEthSpec>::eip4844_only().run(); - } - #[test] fn historical_summary() { SszStaticHandler::::capella_and_later().run(); From d84117c0d050791d7d0440da72c17941785375e1 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Fri, 24 Mar 2023 16:33:49 -0500 Subject: [PATCH 30/96] Removed TODO (#4128) --- beacon_node/store/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 179b099de35..47f0049fc2b 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -160,7 +160,6 @@ pub trait ItemStore: KeyValueStore + Sync + Send + Sized + 'stati pub enum StoreOp<'a, E: EthSpec> { PutBlock(Hash256, Arc>), PutState(Hash256, &'a BeaconState), - // TODO (mark): space can be optimized here by de-duplicating data PutBlobs(Hash256, BlobSidecarList), PutOrphanedBlobsKey(Hash256), PutStateSummary(Hash256, HotStateSummary), From a5addf661c332255453272b7d427ad34677ea83a Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 26 Mar 2023 11:49:16 -0400 Subject: [PATCH 31/96] Rename eip4844 to deneb (#4129) * rename 4844 to deneb * rename 4844 to deneb * move excess data gas field * get EF tests working * fix ef tests lint * fix the blob identifier ef test * fix accessed files ef test script * get beacon chain tests passing --- Makefile | 2 +- .../beacon_chain/src/beacon_block_streamer.rs | 8 +- beacon_node/beacon_chain/src/beacon_chain.rs | 26 +++--- .../beacon_chain/src/blob_verification.rs | 4 +- .../src/data_availability_checker.rs | 20 ++--- .../beacon_chain/src/execution_payload.rs | 2 +- beacon_node/beacon_chain/src/test_utils.rs | 15 ++-- beacon_node/client/src/config.rs | 2 +- beacon_node/execution_layer/src/engine_api.rs | 80 +++++++++--------- .../execution_layer/src/engine_api/http.rs | 6 +- .../src/engine_api/json_structures.rs | 29 +++---- beacon_node/execution_layer/src/lib.rs | 46 +++++----- .../test_utils/execution_block_generator.rs | 22 ++--- .../src/test_utils/handle_rpc.rs | 16 ++-- .../src/test_utils/mock_builder.rs | 4 +- .../src/test_utils/mock_execution_layer.rs | 4 +- .../execution_layer/src/test_utils/mod.rs | 14 +-- .../http_api/src/build_block_contents.rs | 2 +- beacon_node/http_api/src/publish_blocks.rs | 2 +- beacon_node/lighthouse_network/src/config.rs | 2 +- .../lighthouse_network/src/rpc/codec/base.rs | 6 +- .../src/rpc/codec/ssz_snappy.rs | 28 +++--- .../lighthouse_network/src/rpc/protocol.rs | 16 ++-- .../lighthouse_network/src/types/pubsub.rs | 6 +- .../lighthouse_network/src/types/topics.rs | 2 +- .../lighthouse_network/tests/common.rs | 6 +- .../beacon_processor/worker/rpc_methods.rs | 6 +- .../network/src/sync/block_lookups/tests.rs | 8 +- .../network/src/sync/range_sync/batch.rs | 4 +- beacon_node/store/src/hot_cold_store.rs | 16 ++-- .../store/src/impls/execution_payload.rs | 8 +- beacon_node/store/src/partial_beacon_state.rs | 36 ++++---- common/eth2/src/types.rs | 6 +- .../{eip4844 => deneb}/boot_enr.yaml | 0 .../{eip4844 => deneb}/config.yaml | 8 +- .../{eip4844 => deneb}/genesis.ssz.zip | Bin .../gnosis/config.yaml | 6 +- .../mainnet/config.yaml | 6 +- .../sepolia/config.yaml | 6 +- common/eth2_network_config/src/lib.rs | 8 +- consensus/fork_choice/src/fork_choice.rs | 4 +- .../src/common/slash_validator.rs | 2 +- consensus/state_processing/src/genesis.rs | 16 ++-- .../src/per_block_processing.rs | 10 +-- .../{eip4844.rs => deneb.rs} | 2 +- .../{eip4844/eip4844.rs => deneb/deneb.rs} | 2 +- .../process_operations.rs | 2 +- .../src/per_epoch_processing.rs | 2 +- .../src/per_slot_processing.rs | 8 +- consensus/state_processing/src/upgrade.rs | 4 +- .../src/upgrade/{eip4844.rs => deneb.rs} | 12 +-- consensus/types/src/beacon_block.rs | 52 ++++++------ consensus/types/src/beacon_block_body.rs | 44 +++++----- consensus/types/src/beacon_state.rs | 40 ++++----- consensus/types/src/blob_sidecar.rs | 2 +- consensus/types/src/chain_spec.rs | 60 ++++++------- consensus/types/src/consts.rs | 2 +- consensus/types/src/eth_spec.rs | 2 +- consensus/types/src/execution_payload.rs | 22 ++--- .../types/src/execution_payload_header.rs | 40 ++++----- consensus/types/src/fork_context.rs | 6 +- consensus/types/src/fork_name.rs | 32 +++---- consensus/types/src/lib.rs | 14 +-- consensus/types/src/payload.rs | 60 ++++++------- consensus/types/src/signed_beacon_block.rs | 38 ++++----- lcli/src/create_payload_header.rs | 6 +- lcli/src/main.rs | 8 +- lcli/src/new_testnet.rs | 16 ++-- lcli/src/parse_ssz.rs | 6 +- scripts/local_testnet/setup.sh | 6 +- scripts/local_testnet/vars.env | 2 +- scripts/tests/vars.env | 2 +- testing/ef_tests/Makefile | 2 +- testing/ef_tests/check_all_files_accessed.py | 6 +- testing/ef_tests/src/cases/common.rs | 2 +- .../ef_tests/src/cases/epoch_processing.rs | 16 ++-- testing/ef_tests/src/cases/fork.rs | 4 +- .../src/cases/kzg_blob_to_kzg_commitment.rs | 0 .../src/cases/kzg_compute_blob_kzg_proof.rs | 0 .../src/cases/kzg_compute_kzg_proof.rs | 0 .../src/cases/kzg_verify_blob_kzg_proof.rs | 0 .../cases/kzg_verify_blob_kzg_proof_batch.rs | 0 .../src/cases/kzg_verify_kzg_proof.rs | 0 .../src/cases/merkle_proof_validity.rs | 5 ++ testing/ef_tests/src/cases/operations.rs | 2 +- testing/ef_tests/src/cases/transition.rs | 4 +- testing/ef_tests/src/handler.rs | 18 ++-- testing/ef_tests/src/type_name.rs | 9 +- testing/ef_tests/tests/tests.rs | 35 +++++--- .../src/signing_method/web3signer.rs | 6 +- 90 files changed, 572 insertions(+), 549 deletions(-) rename common/eth2_network_config/built_in_network_configs/{eip4844 => deneb}/boot_enr.yaml (100%) rename common/eth2_network_config/built_in_network_configs/{eip4844 => deneb}/config.yaml (89%) rename common/eth2_network_config/built_in_network_configs/{eip4844 => deneb}/genesis.ssz.zip (100%) rename consensus/state_processing/src/per_block_processing/{eip4844.rs => deneb.rs} (67%) rename consensus/state_processing/src/per_block_processing/{eip4844/eip4844.rs => deneb/deneb.rs} (98%) rename consensus/state_processing/src/upgrade/{eip4844.rs => deneb.rs} (89%) create mode 100644 testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs create mode 100644 testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs create mode 100644 testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs create mode 100644 testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs create mode 100644 testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs create mode 100644 testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs diff --git a/Makefile b/Makefile index bf2ad679453..95df2518893 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ PROFILE ?= release # List of all hard forks. This list is used to set env variables for several tests so that # they run for different forks. -FORKS=phase0 altair merge capella eip4844 +FORKS=phase0 altair merge capella deneb # Extra flags for Cargo CARGO_INSTALL_EXTRA_FLAGS?= diff --git a/beacon_node/beacon_chain/src/beacon_block_streamer.rs b/beacon_node/beacon_chain/src/beacon_block_streamer.rs index e56c0c3e399..f626a6d84fd 100644 --- a/beacon_node/beacon_chain/src/beacon_block_streamer.rs +++ b/beacon_node/beacon_chain/src/beacon_block_streamer.rs @@ -3,7 +3,7 @@ use execution_layer::{ExecutionLayer, ExecutionPayloadBodyV1}; use slog::{crit, debug, Logger}; use std::collections::HashMap; use std::sync::Arc; -use store::{DatabaseBlock, ExecutionPayloadEip4844}; +use store::{DatabaseBlock, ExecutionPayloadDeneb}; use task_executor::TaskExecutor; use tokio::sync::{ mpsc::{self, UnboundedSender}, @@ -97,7 +97,7 @@ fn reconstruct_default_header_block( let payload: ExecutionPayload = match fork { ForkName::Merge => ExecutionPayloadMerge::default().into(), ForkName::Capella => ExecutionPayloadCapella::default().into(), - ForkName::Eip4844 => ExecutionPayloadEip4844::default().into(), + ForkName::Deneb => ExecutionPayloadDeneb::default().into(), ForkName::Base | ForkName::Altair => { return Err(Error::PayloadReconstruction(format!( "Block with fork variant {} has execution payload", @@ -726,6 +726,8 @@ mod tests { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = Some(Epoch::new(bellatrix_fork_epoch as u64)); spec.capella_fork_epoch = Some(Epoch::new(capella_fork_epoch as u64)); + //FIXME(sean) extend this to test deneb? + spec.deneb_fork_epoch = None; let harness = get_harness(VALIDATOR_COUNT, spec); // go to bellatrix fork @@ -845,6 +847,8 @@ mod tests { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = Some(Epoch::new(bellatrix_fork_epoch as u64)); spec.capella_fork_epoch = Some(Epoch::new(capella_fork_epoch as u64)); + //FIXME(sean) extend this to test deneb? + spec.deneb_fork_epoch = None; let harness = get_harness(VALIDATOR_COUNT, spec); diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 1d4f1d17e97..706db639606 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -116,7 +116,7 @@ use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; use types::blob_sidecar::{BlobIdentifier, BlobSidecarList, Blobs}; -use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; +use types::consts::deneb::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; use types::consts::merge::INTERVALS_PER_SLOT; use types::*; @@ -1107,7 +1107,7 @@ impl BeaconChain { /// ## Errors /// - any database read errors /// - block and blobs are inconsistent in the database - /// - this method is called with a pre-eip4844 block root + /// - this method is called with a pre-deneb block root /// - this method is called for a blob that is beyond the prune depth pub fn get_blobs( &self, @@ -4465,7 +4465,7 @@ impl BeaconChain { // allows it to run concurrently with things like attestation packing. let prepare_payload_handle = match &state { BeaconState::Base(_) | BeaconState::Altair(_) => None, - BeaconState::Merge(_) | BeaconState::Capella(_) | BeaconState::Eip4844(_) => { + BeaconState::Merge(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) => { let prepare_payload_handle = get_execution_payload(self.clone(), &state, proposer_index, builder_params)?; Some(prepare_payload_handle) @@ -4773,17 +4773,17 @@ impl BeaconChain { None, ) } - BeaconState::Eip4844(_) => { + BeaconState::Deneb(_) => { let (payload, kzg_commitments, blobs) = block_contents .ok_or(BlockProductionError::MissingExecutionPayload)? .deconstruct(); ( - BeaconBlock::Eip4844(BeaconBlockEip4844 { + BeaconBlock::Deneb(BeaconBlockDeneb { slot, proposer_index, parent_root, state_root: Hash256::zero(), - body: BeaconBlockBodyEip4844 { + body: BeaconBlockBodyDeneb { randao_reveal, eth1_data, graffiti, @@ -4862,7 +4862,7 @@ impl BeaconChain { let beacon_block_root = block.canonical_root(); let expected_kzg_commitments = block.body().blob_kzg_commitments().map_err(|_| { BlockProductionError::InvalidBlockVariant( - "EIP4844 block does not contain kzg commitments".to_string(), + "DENEB block does not contain kzg commitments".to_string(), ) })?; @@ -5162,7 +5162,7 @@ impl BeaconChain { } else { let withdrawals = match self.spec.fork_name_at_slot::(prepare_slot) { ForkName::Base | ForkName::Altair | ForkName::Merge => None, - ForkName::Capella | ForkName::Eip4844 => { + ForkName::Capella | ForkName::Deneb => { let chain = self.clone(); self.spawn_blocking_handle( move || { @@ -6185,9 +6185,9 @@ impl BeaconChain { } /// The epoch at which we require a data availability check in block processing. - /// `None` if the `Eip4844` fork is disabled. + /// `None` if the `Deneb` fork is disabled. pub fn data_availability_boundary(&self) -> Option { - self.spec.eip4844_fork_epoch.and_then(|fork_epoch| { + self.spec.deneb_fork_epoch.and_then(|fork_epoch| { self.epoch().ok().map(|current_epoch| { std::cmp::max( fork_epoch, @@ -6203,13 +6203,13 @@ impl BeaconChain { .map_or(false, |da_epoch| block_epoch >= da_epoch) } - /// Returns `true` if we are at or past the `Eip4844` fork. This will always return `false` if - /// the `Eip4844` fork is disabled. + /// Returns `true` if we are at or past the `Deneb` fork. This will always return `false` if + /// the `Deneb` fork is disabled. pub fn is_data_availability_check_required(&self) -> Result { let current_epoch = self.epoch()?; Ok(self .spec - .eip4844_fork_epoch + .deneb_fork_epoch .map(|fork_epoch| fork_epoch <= current_epoch) .unwrap_or(false)) } diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 9d1a7c708ec..583d1a60b31 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -64,7 +64,7 @@ pub enum BlobError { BeaconChainError(BeaconChainError), /// No blobs for the specified block where we would expect blobs. UnavailableBlobs, - /// Blobs provided for a pre-Eip4844 fork. + /// Blobs provided for a pre-Deneb fork. InconsistentFork, /// The `blobs_sidecar.message.beacon_block_root` block is unknown. @@ -337,7 +337,7 @@ pub type KzgVerifiedBlobList = Vec>; #[derive(Debug, Clone)] pub enum MaybeAvailableBlock { /// This variant is fully available. - /// i.e. for pre-eip4844 blocks, it contains a (`SignedBeaconBlock`, `Blobs::None`) and for + /// i.e. for pre-deneb blocks, it contains a (`SignedBeaconBlock`, `Blobs::None`) and for /// post-4844 blocks, it contains a `SignedBeaconBlock` and a Blobs variant other than `Blobs::None`. Available(AvailableBlock), /// This variant is not fully available and requires blobs to become fully available. diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 42e97a974e4..b6c53e354e1 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -9,13 +9,13 @@ use kzg::Kzg; use parking_lot::{Mutex, RwLock}; use slot_clock::SlotClock; use ssz_types::{Error, VariableList}; -use state_processing::per_block_processing::eip4844::eip4844::verify_kzg_commitments_against_transactions; +use state_processing::per_block_processing::deneb::deneb::verify_kzg_commitments_against_transactions; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::sync::Arc; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::{BlobIdentifier, BlobSidecar}; -use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; +use types::consts::deneb::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; use types::{ BeaconBlockRef, BlobSidecarList, ChainSpec, Epoch, EthSpec, ExecPayload, FullPayload, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, @@ -261,7 +261,7 @@ impl DataAvailabilityChecker { let blobs = match blob_requirements { BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs, BlobRequirements::NotRequired => VerifiedBlobs::NotRequired, - BlobRequirements::PreEip4844 => VerifiedBlobs::PreEip4844, + BlobRequirements::PreDeneb => VerifiedBlobs::PreDeneb, BlobRequirements::Required => return Err(AvailabilityCheckError::MissingBlobs), }; Ok(AvailableBlock { block, blobs }) @@ -295,7 +295,7 @@ impl DataAvailabilityChecker { let blobs = match blob_requirements { BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs, BlobRequirements::NotRequired => VerifiedBlobs::NotRequired, - BlobRequirements::PreEip4844 => VerifiedBlobs::PreEip4844, + BlobRequirements::PreDeneb => VerifiedBlobs::PreDeneb, BlobRequirements::Required => { return Ok(MaybeAvailableBlock::AvailabilityPending( AvailabilityPendingBlock { block }, @@ -371,15 +371,15 @@ impl DataAvailabilityChecker { BlobRequirements::NotRequired } } else { - BlobRequirements::PreEip4844 + BlobRequirements::PreDeneb }; Ok(verified_blobs) } /// The epoch at which we require a data availability check in block processing. - /// `None` if the `Eip4844` fork is disabled. + /// `None` if the `Deneb` fork is disabled. pub fn data_availability_boundary(&self) -> Option { - self.spec.eip4844_fork_epoch.and_then(|fork_epoch| { + self.spec.deneb_fork_epoch.and_then(|fork_epoch| { self.slot_clock .now() .map(|slot| slot.epoch(T::slots_per_epoch())) @@ -407,7 +407,7 @@ pub enum BlobRequirements { /// The block's `kzg_commitments` field is empty so it does not contain any blobs. EmptyBlobs, /// This is a block prior to the 4844 fork, so doesn't require any blobs - PreEip4844, + PreDeneb, } #[derive(Clone, Debug, PartialEq)] @@ -447,7 +447,7 @@ impl AvailableBlock { pub fn deconstruct(self) -> (Arc>, Option>) { match self.blobs { - VerifiedBlobs::EmptyBlobs | VerifiedBlobs::NotRequired | VerifiedBlobs::PreEip4844 => { + VerifiedBlobs::EmptyBlobs | VerifiedBlobs::NotRequired | VerifiedBlobs::PreDeneb => { (self.block, None) } VerifiedBlobs::Available(blobs) => (self.block, Some(blobs)), @@ -465,7 +465,7 @@ pub enum VerifiedBlobs { /// The block's `kzg_commitments` field is empty so it does not contain any blobs. EmptyBlobs, /// This is a block prior to the 4844 fork, so doesn't require any blobs - PreEip4844, + PreDeneb, } impl AsBlock for AvailableBlock { diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index 41baa956ab9..51c7ec29901 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -419,7 +419,7 @@ pub fn get_execution_payload< let latest_execution_payload_header_block_hash = state.latest_execution_payload_header()?.block_hash(); let withdrawals = match state { - &BeaconState::Capella(_) | &BeaconState::Eip4844(_) => { + &BeaconState::Capella(_) | &BeaconState::Deneb(_) => { Some(get_expected_withdrawals(state, spec)?.into()) } &BeaconState::Merge(_) => None, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index df2545adf12..c54c3df656b 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -431,10 +431,9 @@ where spec.capella_fork_epoch.map(|epoch| { genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); - mock.server.execution_block_generator().eip4844_time = - spec.eip4844_fork_epoch.map(|epoch| { - genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() - }); + mock.server.execution_block_generator().deneb_time = spec.deneb_fork_epoch.map(|epoch| { + genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() + }); self } @@ -444,14 +443,14 @@ where let shanghai_time = spec.capella_fork_epoch.map(|epoch| { HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); - let eip4844_time = spec.eip4844_fork_epoch.map(|epoch| { + let deneb_time = spec.deneb_fork_epoch.map(|epoch| { HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); let mock = MockExecutionLayer::new( self.runtime.task_executor.clone(), DEFAULT_TERMINAL_BLOCK, shanghai_time, - eip4844_time, + deneb_time, None, Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec, @@ -475,14 +474,14 @@ where let shanghai_time = spec.capella_fork_epoch.map(|epoch| { HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); - let eip4844_time = spec.eip4844_fork_epoch.map(|epoch| { + let deneb_time = spec.deneb_fork_epoch.map(|epoch| { HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); let mock_el = MockExecutionLayer::new( self.runtime.task_executor.clone(), DEFAULT_TERMINAL_BLOCK, shanghai_time, - eip4844_time, + deneb_time, builder_threshold, Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec.clone(), diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index f9d26d271d5..8ce0fa2cc5f 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -52,7 +52,7 @@ pub struct Config { /// Path where the blobs database will be located if blobs should be in a separate database. /// /// The capacity this location should hold varies with the data availability boundary. It - /// should be able to store < 69 GB when [MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS](types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS) is 4096 + /// should be able to store < 69 GB when [MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS](types::consts::deneb::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS) is 4096 /// epochs of 32 slots (up to 131072 bytes data per blob and up to 4 blobs per block, 88 bytes /// of [BlobsSidecar](types::BlobsSidecar) metadata per block). pub blobs_db_path: Option, diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 009183d7ab9..02d9e814988 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -21,7 +21,7 @@ pub use types::{ ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, Withdrawal, Withdrawals, }; -use types::{ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge}; +use types::{ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge}; pub mod auth; pub mod http; @@ -150,7 +150,7 @@ pub struct ExecutionBlock { /// Representation of an execution block with enough detail to reconstruct a payload. #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Deneb), variant_attributes( derive(Clone, Debug, PartialEq, Serialize, Deserialize,), serde(bound = "T: EthSpec", rename_all = "camelCase"), @@ -181,14 +181,14 @@ pub struct ExecutionBlockWithTransactions { #[serde(with = "ssz_types::serde_utils::hex_var_list")] pub extra_data: VariableList, pub base_fee_per_gas: Uint256, - #[superstruct(only(Eip4844))] - #[serde(with = "eth2_serde_utils::u256_hex_be")] - pub excess_data_gas: Uint256, #[serde(rename = "hash")] pub block_hash: ExecutionBlockHash, pub transactions: Vec, - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Deneb))] pub withdrawals: Vec, + #[superstruct(only(Deneb))] + #[serde(with = "eth2_serde_utils::u256_hex_be")] + pub excess_data_gas: Uint256, } impl TryFrom> for ExecutionBlockWithTransactions { @@ -242,33 +242,31 @@ impl TryFrom> for ExecutionBlockWithTransactions .collect(), }) } - ExecutionPayload::Eip4844(block) => { - Self::Eip4844(ExecutionBlockWithTransactionsEip4844 { - parent_hash: block.parent_hash, - fee_recipient: block.fee_recipient, - state_root: block.state_root, - receipts_root: block.receipts_root, - logs_bloom: block.logs_bloom, - prev_randao: block.prev_randao, - block_number: block.block_number, - gas_limit: block.gas_limit, - gas_used: block.gas_used, - timestamp: block.timestamp, - extra_data: block.extra_data, - base_fee_per_gas: block.base_fee_per_gas, - excess_data_gas: block.excess_data_gas, - block_hash: block.block_hash, - transactions: block - .transactions - .iter() - .map(|tx| Transaction::decode(&Rlp::new(tx))) - .collect::, _>>()?, - withdrawals: Vec::from(block.withdrawals) - .into_iter() - .map(|withdrawal| withdrawal.into()) - .collect(), - }) - } + ExecutionPayload::Deneb(block) => Self::Deneb(ExecutionBlockWithTransactionsDeneb { + parent_hash: block.parent_hash, + fee_recipient: block.fee_recipient, + state_root: block.state_root, + receipts_root: block.receipts_root, + logs_bloom: block.logs_bloom, + prev_randao: block.prev_randao, + block_number: block.block_number, + gas_limit: block.gas_limit, + gas_used: block.gas_used, + timestamp: block.timestamp, + extra_data: block.extra_data, + base_fee_per_gas: block.base_fee_per_gas, + block_hash: block.block_hash, + transactions: block + .transactions + .iter() + .map(|tx| Transaction::decode(&Rlp::new(tx))) + .collect::, _>>()?, + withdrawals: Vec::from(block.withdrawals) + .into_iter() + .map(|withdrawal| withdrawal.into()) + .collect(), + excess_data_gas: block.excess_data_gas, + }), }; Ok(json_payload) } @@ -363,7 +361,7 @@ pub struct ProposeBlindedBlockResponse { } #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Deneb), variant_attributes(derive(Clone, Debug, PartialEq),), map_into(ExecutionPayload), map_ref_into(ExecutionPayloadRef), @@ -376,8 +374,8 @@ pub struct GetPayloadResponse { pub execution_payload: ExecutionPayloadMerge, #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] pub execution_payload: ExecutionPayloadCapella, - #[superstruct(only(Eip4844), partial_getter(rename = "execution_payload_eip4844"))] - pub execution_payload: ExecutionPayloadEip4844, + #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] + pub execution_payload: ExecutionPayloadDeneb, pub block_value: Uint256, } @@ -408,8 +406,8 @@ impl From> for (ExecutionPayload, Uint256) ExecutionPayload::Capella(inner.execution_payload), inner.block_value, ), - GetPayloadResponse::Eip4844(inner) => ( - ExecutionPayload::Eip4844(inner.execution_payload), + GetPayloadResponse::Deneb(inner) => ( + ExecutionPayload::Deneb(inner.execution_payload), inner.block_value, ), } @@ -484,9 +482,9 @@ impl ExecutionPayloadBodyV1 { )) } } - ExecutionPayloadHeader::Eip4844(header) => { + ExecutionPayloadHeader::Deneb(header) => { if let Some(withdrawals) = self.withdrawals { - Ok(ExecutionPayload::Eip4844(ExecutionPayloadEip4844 { + Ok(ExecutionPayload::Deneb(ExecutionPayloadDeneb { parent_hash: header.parent_hash, fee_recipient: header.fee_recipient, state_root: header.state_root, @@ -499,10 +497,10 @@ impl ExecutionPayloadBodyV1 { timestamp: header.timestamp, extra_data: header.extra_data, base_fee_per_gas: header.base_fee_per_gas, - excess_data_gas: header.excess_data_gas, block_hash: header.block_hash, transactions: self.transactions, withdrawals, + excess_data_gas: header.excess_data_gas, })) } else { Err(format!( diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index df7d5d25e8c..0daff8cf34a 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -757,7 +757,7 @@ impl HttpJsonRpc { ) .await?, ), - ForkName::Eip4844 => ExecutionBlockWithTransactions::Eip4844( + ForkName::Deneb => ExecutionBlockWithTransactions::Deneb( self.rpc_request( ETH_GET_BLOCK_BY_HASH, params, @@ -876,7 +876,7 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V2(response).into()) } - ForkName::Base | ForkName::Altair | ForkName::Eip4844 => Err( + ForkName::Base | ForkName::Altair | ForkName::Deneb => Err( Error::UnsupportedForkVariant(format!("called get_payload_v2 with {}", fork_name)), ), } @@ -910,7 +910,7 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V2(response).into()) } - ForkName::Eip4844 => { + ForkName::Deneb => { let response: JsonGetPayloadResponseV3 = self .rpc_request( ENGINE_GET_PAYLOAD_V3, diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 45ec4862605..d7d9aae2987 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -5,9 +5,8 @@ use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; use types::{ - EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, - ExecutionPayloadEip4844, ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, - VariableList, Withdrawal, + EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, + ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, VariableList, Withdrawal, }; #[derive(Debug, PartialEq, Serialize, Deserialize)] @@ -94,14 +93,14 @@ pub struct JsonExecutionPayload { pub extra_data: VariableList, #[serde(with = "eth2_serde_utils::u256_hex_be")] pub base_fee_per_gas: Uint256, - #[superstruct(only(V3))] - #[serde(with = "eth2_serde_utils::u256_hex_be")] - pub excess_data_gas: Uint256, pub block_hash: ExecutionBlockHash, #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, #[superstruct(only(V2, V3))] pub withdrawals: VariableList, + #[superstruct(only(V3))] + #[serde(with = "eth2_serde_utils::u256_hex_be")] + pub excess_data_gas: Uint256, } impl From> for JsonExecutionPayloadV1 { @@ -150,8 +149,8 @@ impl From> for JsonExecutionPayloadV2 } } } -impl From> for JsonExecutionPayloadV3 { - fn from(payload: ExecutionPayloadEip4844) -> Self { +impl From> for JsonExecutionPayloadV3 { + fn from(payload: ExecutionPayloadDeneb) -> Self { JsonExecutionPayloadV3 { parent_hash: payload.parent_hash, fee_recipient: payload.fee_recipient, @@ -165,7 +164,6 @@ impl From> for JsonExecutionPayloadV3 timestamp: payload.timestamp, extra_data: payload.extra_data, base_fee_per_gas: payload.base_fee_per_gas, - excess_data_gas: payload.excess_data_gas, block_hash: payload.block_hash, transactions: payload.transactions, withdrawals: payload @@ -174,6 +172,7 @@ impl From> for JsonExecutionPayloadV3 .map(Into::into) .collect::>() .into(), + excess_data_gas: payload.excess_data_gas, } } } @@ -183,7 +182,7 @@ impl From> for JsonExecutionPayload { match execution_payload { ExecutionPayload::Merge(payload) => JsonExecutionPayload::V1(payload.into()), ExecutionPayload::Capella(payload) => JsonExecutionPayload::V2(payload.into()), - ExecutionPayload::Eip4844(payload) => JsonExecutionPayload::V3(payload.into()), + ExecutionPayload::Deneb(payload) => JsonExecutionPayload::V3(payload.into()), } } } @@ -234,9 +233,9 @@ impl From> for ExecutionPayloadCapella } } } -impl From> for ExecutionPayloadEip4844 { +impl From> for ExecutionPayloadDeneb { fn from(payload: JsonExecutionPayloadV3) -> Self { - ExecutionPayloadEip4844 { + ExecutionPayloadDeneb { parent_hash: payload.parent_hash, fee_recipient: payload.fee_recipient, state_root: payload.state_root, @@ -249,7 +248,6 @@ impl From> for ExecutionPayloadEip4844 timestamp: payload.timestamp, extra_data: payload.extra_data, base_fee_per_gas: payload.base_fee_per_gas, - excess_data_gas: payload.excess_data_gas, block_hash: payload.block_hash, transactions: payload.transactions, withdrawals: payload @@ -258,6 +256,7 @@ impl From> for ExecutionPayloadEip4844 .map(Into::into) .collect::>() .into(), + excess_data_gas: payload.excess_data_gas, } } } @@ -267,7 +266,7 @@ impl From> for ExecutionPayload { match json_execution_payload { JsonExecutionPayload::V1(payload) => ExecutionPayload::Merge(payload.into()), JsonExecutionPayload::V2(payload) => ExecutionPayload::Capella(payload.into()), - JsonExecutionPayload::V3(payload) => ExecutionPayload::Eip4844(payload.into()), + JsonExecutionPayload::V3(payload) => ExecutionPayload::Deneb(payload.into()), } } } @@ -310,7 +309,7 @@ impl From> for GetPayloadResponse { }) } JsonGetPayloadResponse::V3(response) => { - GetPayloadResponse::Eip4844(GetPayloadResponseEip4844 { + GetPayloadResponse::Deneb(GetPayloadResponseDeneb { execution_payload: response.execution_payload.into(), block_value: response.block_value, }) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 9704801d766..a500bf9ee50 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -43,13 +43,13 @@ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; -use types::consts::eip4844::BLOB_TX_TYPE; +use types::consts::deneb::BLOB_TX_TYPE; use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlobTransaction}; use types::Withdrawals; use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, - ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, ForkName, + ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, ForkName, }; use types::{ ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Transaction, @@ -208,7 +208,7 @@ impl> BlockProposalContents BlockProposalContents::PayloadAndBlobs { + ForkName::Deneb => BlockProposalContents::PayloadAndBlobs { payload: Payload::default_at_fork(fork_name)?, block_value: Uint256::zero(), blobs: VariableList::default(), @@ -1111,7 +1111,7 @@ impl ExecutionLayer { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { None } - ForkName::Eip4844 => { + ForkName::Deneb => { debug!( self.log(), "Issuing engine_getBlobsBundle"; @@ -1703,7 +1703,7 @@ impl ExecutionLayer { return match fork { ForkName::Merge => Ok(Some(ExecutionPayloadMerge::default().into())), ForkName::Capella => Ok(Some(ExecutionPayloadCapella::default().into())), - ForkName::Eip4844 => Ok(Some(ExecutionPayloadEip4844::default().into())), + ForkName::Deneb => Ok(Some(ExecutionPayloadDeneb::default().into())), ForkName::Base | ForkName::Altair => Err(ApiError::UnsupportedForkVariant( format!("called get_payload_by_block_hash_from_engine with {}", fork), )), @@ -1776,32 +1776,32 @@ impl ExecutionLayer { withdrawals, }) } - ExecutionBlockWithTransactions::Eip4844(eip4844_block) => { + ExecutionBlockWithTransactions::Deneb(deneb_block) => { let withdrawals = VariableList::new( - eip4844_block + deneb_block .withdrawals .into_iter() .map(Into::into) .collect(), ) .map_err(ApiError::DeserializeWithdrawals)?; - ExecutionPayload::Eip4844(ExecutionPayloadEip4844 { - parent_hash: eip4844_block.parent_hash, - fee_recipient: eip4844_block.fee_recipient, - state_root: eip4844_block.state_root, - receipts_root: eip4844_block.receipts_root, - logs_bloom: eip4844_block.logs_bloom, - prev_randao: eip4844_block.prev_randao, - block_number: eip4844_block.block_number, - gas_limit: eip4844_block.gas_limit, - gas_used: eip4844_block.gas_used, - timestamp: eip4844_block.timestamp, - extra_data: eip4844_block.extra_data, - base_fee_per_gas: eip4844_block.base_fee_per_gas, - excess_data_gas: eip4844_block.excess_data_gas, - block_hash: eip4844_block.block_hash, - transactions: convert_transactions(eip4844_block.transactions)?, + ExecutionPayload::Deneb(ExecutionPayloadDeneb { + parent_hash: deneb_block.parent_hash, + fee_recipient: deneb_block.fee_recipient, + state_root: deneb_block.state_root, + receipts_root: deneb_block.receipts_root, + logs_bloom: deneb_block.logs_bloom, + prev_randao: deneb_block.prev_randao, + block_number: deneb_block.block_number, + gas_limit: deneb_block.gas_limit, + gas_used: deneb_block.gas_used, + timestamp: deneb_block.timestamp, + extra_data: deneb_block.extra_data, + base_fee_per_gas: deneb_block.base_fee_per_gas, + block_hash: deneb_block.block_hash, + transactions: convert_transactions(deneb_block.transactions)?, withdrawals, + excess_data_gas: deneb_block.excess_data_gas, }) } }; diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index fe3c6f274b8..e5a5c70d7d7 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -13,8 +13,8 @@ use std::collections::HashMap; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use types::{ - EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, - ExecutionPayloadEip4844, ExecutionPayloadMerge, ForkName, Hash256, Uint256, + EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, + ExecutionPayloadMerge, ForkName, Hash256, Uint256, }; const GAS_LIMIT: u64 = 16384; @@ -118,7 +118,7 @@ pub struct ExecutionBlockGenerator { * Post-merge fork triggers */ pub shanghai_time: Option, // withdrawals - pub eip4844_time: Option, // 4844 + pub deneb_time: Option, // 4844 } impl ExecutionBlockGenerator { @@ -127,7 +127,7 @@ impl ExecutionBlockGenerator { terminal_block_number: u64, terminal_block_hash: ExecutionBlockHash, shanghai_time: Option, - eip4844_time: Option, + deneb_time: Option, ) -> Self { let mut gen = Self { head_block: <_>::default(), @@ -141,7 +141,7 @@ impl ExecutionBlockGenerator { next_payload_id: 0, payload_ids: <_>::default(), shanghai_time, - eip4844_time, + deneb_time, }; gen.insert_pow_block(0).unwrap(); @@ -174,8 +174,8 @@ impl ExecutionBlockGenerator { } pub fn get_fork_at_timestamp(&self, timestamp: u64) -> ForkName { - match self.eip4844_time { - Some(fork_time) if timestamp >= fork_time => ForkName::Eip4844, + match self.deneb_time { + Some(fork_time) if timestamp >= fork_time => ForkName::Deneb, _ => match self.shanghai_time { Some(fork_time) if timestamp >= fork_time => ForkName::Capella, _ => ForkName::Merge, @@ -535,8 +535,8 @@ impl ExecutionBlockGenerator { withdrawals: pa.withdrawals.clone().into(), }) } - ForkName::Eip4844 => { - ExecutionPayload::Eip4844(ExecutionPayloadEip4844 { + ForkName::Deneb => { + ExecutionPayload::Deneb(ExecutionPayloadDeneb { parent_hash: forkchoice_state.head_block_hash, fee_recipient: pa.suggested_fee_recipient, receipts_root: Hash256::repeat_byte(42), @@ -549,11 +549,11 @@ impl ExecutionBlockGenerator { timestamp: pa.timestamp, extra_data: "block gen was here".as_bytes().to_vec().into(), base_fee_per_gas: Uint256::one(), - // FIXME(4844): maybe this should be set to something? - excess_data_gas: Uint256::one(), block_hash: ExecutionBlockHash::zero(), transactions: vec![].into(), withdrawals: pa.withdrawals.clone().into(), + // FIXME(deneb) maybe this should be set to something? + excess_data_gas: Uint256::one(), }) } _ => unreachable!(), diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index a1488e2dc9a..aae1a0b8989 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -149,17 +149,17 @@ pub async fn handle_rpc( )); } } - ForkName::Eip4844 => { + ForkName::Deneb => { if method == ENGINE_NEW_PAYLOAD_V1 || method == ENGINE_NEW_PAYLOAD_V2 { return Err(( - format!("{} called after eip4844 fork!", method), + format!("{} called after deneb fork!", method), GENERIC_ERROR_CODE, )); } if matches!(request, JsonExecutionPayload::V1(_)) { return Err(( format!( - "{} called with `ExecutionPayloadV1` after eip4844 fork!", + "{} called with `ExecutionPayloadV1` after deneb fork!", method ), GENERIC_ERROR_CODE, @@ -168,7 +168,7 @@ pub async fn handle_rpc( if matches!(request, JsonExecutionPayload::V2(_)) { return Err(( format!( - "{} called with `ExecutionPayloadV2` after eip4844 fork!", + "{} called with `ExecutionPayloadV2` after deneb fork!", method ), GENERIC_ERROR_CODE, @@ -237,16 +237,16 @@ pub async fn handle_rpc( FORK_REQUEST_MISMATCH_ERROR_CODE, )); } - // validate method called correctly according to eip4844 fork time + // validate method called correctly according to deneb fork time if ctx .execution_block_generator .read() .get_fork_at_timestamp(response.timestamp()) - == ForkName::Eip4844 + == ForkName::Deneb && (method == ENGINE_GET_PAYLOAD_V1 || method == ENGINE_GET_PAYLOAD_V2) { return Err(( - format!("{} called after eip4844 fork!", method), + format!("{} called after deneb fork!", method), FORK_REQUEST_MISMATCH_ERROR_CODE, )); } @@ -357,7 +357,7 @@ pub async fn handle_rpc( )); } } - ForkName::Capella | ForkName::Eip4844 => { + ForkName::Capella | ForkName::Deneb => { if method == ENGINE_FORKCHOICE_UPDATED_V1 { return Err(( format!("{} called after Capella fork!", method), diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 19972650139..fe5414028cd 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -405,7 +405,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { let payload_attributes = match fork { ForkName::Merge => PayloadAttributes::new(timestamp, *prev_randao, fee_recipient, None), // the withdrawals root is filled in by operations - ForkName::Capella | ForkName::Eip4844 => { + ForkName::Capella | ForkName::Deneb => { PayloadAttributes::new(timestamp, *prev_randao, fee_recipient, Some(vec![])) } ForkName::Base | ForkName::Altair => { @@ -452,7 +452,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { value: to_ssz_rs(&Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI))?, public_key: self.builder_sk.public_key(), }), - ForkName::Base | ForkName::Altair | ForkName::Eip4844 => { + ForkName::Base | ForkName::Altair | ForkName::Deneb => { return Err(BlindedBlockProviderError::Custom(format!( "Unsupported fork: {}", fork diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 1a5d1fd1983..44fc2a5ec2d 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -41,7 +41,7 @@ impl MockExecutionLayer { executor: TaskExecutor, terminal_block: u64, shanghai_time: Option, - eip4844_time: Option, + deneb_time: Option, builder_threshold: Option, jwt_key: Option, spec: ChainSpec, @@ -57,7 +57,7 @@ impl MockExecutionLayer { terminal_block, spec.terminal_block_hash, shanghai_time, - eip4844_time, + deneb_time, ); let url = SensitiveUrl::parse(&server.url()).unwrap(); diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 3c0763a8fb9..60f5bf341f6 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -62,7 +62,7 @@ pub struct MockExecutionConfig { pub terminal_block: u64, pub terminal_block_hash: ExecutionBlockHash, pub shanghai_time: Option, - pub eip4844_time: Option, + pub deneb_time: Option, } impl Default for MockExecutionConfig { @@ -74,7 +74,7 @@ impl Default for MockExecutionConfig { terminal_block_hash: ExecutionBlockHash::zero(), server_config: Config::default(), shanghai_time: None, - eip4844_time: None, + deneb_time: None, } } } @@ -95,7 +95,7 @@ impl MockServer { DEFAULT_TERMINAL_BLOCK, ExecutionBlockHash::zero(), None, // FIXME(capella): should this be the default? - None, // FIXME(eip4844): should this be the default? + None, // FIXME(deneb): should this be the default? ) } @@ -107,7 +107,7 @@ impl MockServer { terminal_block_hash, server_config, shanghai_time, - eip4844_time, + deneb_time, } = config; let last_echo_request = Arc::new(RwLock::new(None)); let preloaded_responses = Arc::new(Mutex::new(vec![])); @@ -116,7 +116,7 @@ impl MockServer { terminal_block, terminal_block_hash, shanghai_time, - eip4844_time, + deneb_time, ); let ctx: Arc> = Arc::new(Context { @@ -175,7 +175,7 @@ impl MockServer { terminal_block: u64, terminal_block_hash: ExecutionBlockHash, shanghai_time: Option, - eip4844_time: Option, + deneb_time: Option, ) -> Self { Self::new_with_config( handle, @@ -186,7 +186,7 @@ impl MockServer { terminal_block, terminal_block_hash, shanghai_time, - eip4844_time, + deneb_time, }, ) } diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index d40fef1d908..d6c5ada0071 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -14,7 +14,7 @@ pub fn build_block_contents { Ok(BlockContents::Block(block)) } - ForkName::Eip4844 => { + ForkName::Deneb => { let block_root = &block.canonical_root(); if let Some(blob_sidecars) = chain.proposal_blob_cache.pop(block_root) { let block_and_blobs = BeaconBlockAndBlobSidecars { diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index c470686ad7f..4894663225e 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -68,7 +68,7 @@ pub async fn publish_block( crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; block.into() } - SignedBeaconBlock::Eip4844(_) => { + SignedBeaconBlock::Deneb(_) => { crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; if let Some(signed_blobs) = maybe_blobs { for (blob_index, blob) in signed_blobs.clone().into_iter().enumerate() { diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 79c8b67d75a..d74d9098e92 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -412,7 +412,7 @@ pub fn gossipsub_config(network_load: u8, fork_context: Arc) -> Gos match fork_context.current_fork() { // according to: https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/p2p-interface.md#the-gossip-domain-gossipsub // the derivation of the message-id remains the same in the merge and for eip 4844. - ForkName::Altair | ForkName::Merge | ForkName::Capella | ForkName::Eip4844 => { + ForkName::Altair | ForkName::Merge | ForkName::Capella | ForkName::Deneb => { let topic_len_bytes = topic_bytes.len().to_le_bytes(); let mut vec = Vec::with_capacity( prefix.len() + topic_len_bytes.len() + topic_bytes.len() + message.data.len(), diff --git a/beacon_node/lighthouse_network/src/rpc/codec/base.rs b/beacon_node/lighthouse_network/src/rpc/codec/base.rs index 164a7c025d9..c2ad2320d9f 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/base.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/base.rs @@ -194,19 +194,19 @@ mod tests { let altair_fork_epoch = Epoch::new(1); let merge_fork_epoch = Epoch::new(2); let capella_fork_epoch = Epoch::new(3); - let eip4844_fork_epoch = Epoch::new(4); + let deneb_fork_epoch = Epoch::new(4); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); chain_spec.bellatrix_fork_epoch = Some(merge_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); - chain_spec.eip4844_fork_epoch = Some(eip4844_fork_epoch); + chain_spec.deneb_fork_epoch = Some(deneb_fork_epoch); let current_slot = match fork_name { ForkName::Base => Slot::new(0), ForkName::Altair => altair_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Merge => merge_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Capella => capella_fork_epoch.start_slot(Spec::slots_per_epoch()), - ForkName::Eip4844 => eip4844_fork_epoch.start_slot(Spec::slots_per_epoch()), + ForkName::Deneb => deneb_fork_epoch.start_slot(Spec::slots_per_epoch()), }; ForkContext::new::(current_slot, Hash256::zero(), &chain_spec) } diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index ab61f45be94..e9622f24a48 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -18,7 +18,7 @@ use tokio_util::codec::{Decoder, Encoder}; use types::{light_client_bootstrap::LightClientBootstrap, BlobSidecar}; use types::{ EthSpec, ForkContext, ForkName, Hash256, SignedBeaconBlock, SignedBeaconBlockAltair, - SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockEip4844, + SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockMerge, }; use unsigned_varint::codec::Uvi; @@ -419,9 +419,9 @@ fn context_bytes( return match **ref_box_block { // NOTE: If you are adding another fork type here, be sure to modify the // `fork_context.to_context_bytes()` function to support it as well! - SignedBeaconBlock::Eip4844 { .. } => { - // Eip4844 context being `None` implies that "merge never happened". - fork_context.to_context_bytes(ForkName::Eip4844) + SignedBeaconBlock::Deneb { .. } => { + // Deneb context being `None` implies that "merge never happened". + fork_context.to_context_bytes(ForkName::Deneb) } SignedBeaconBlock::Capella { .. } => { // Capella context being `None` implies that "merge never happened". @@ -440,7 +440,7 @@ fn context_bytes( }; } if let RPCResponse::BlobsByRange(_) | RPCResponse::SidecarByRoot(_) = rpc_variant { - return fork_context.to_context_bytes(ForkName::Eip4844); + return fork_context.to_context_bytes(ForkName::Deneb); } } } @@ -580,7 +580,7 @@ fn handle_v1_response( ) })?; match fork_name { - ForkName::Eip4844 => Ok(Some(RPCResponse::BlobsByRange(Arc::new( + ForkName::Deneb => Ok(Some(RPCResponse::BlobsByRange(Arc::new( BlobSidecar::from_ssz_bytes(decoded_buffer)?, )))), _ => Err(RPCError::ErrorResponse( @@ -597,7 +597,7 @@ fn handle_v1_response( ) })?; match fork_name { - ForkName::Eip4844 => Ok(Some(RPCResponse::SidecarByRoot(Arc::new( + ForkName::Deneb => Ok(Some(RPCResponse::SidecarByRoot(Arc::new( BlobSidecar::from_ssz_bytes(decoded_buffer)?, )))), _ => Err(RPCError::ErrorResponse( @@ -662,8 +662,8 @@ fn handle_v2_response( decoded_buffer, )?), )))), - ForkName::Eip4844 => Ok(Some(RPCResponse::BlocksByRange(Arc::new( - SignedBeaconBlock::Eip4844(SignedBeaconBlockEip4844::from_ssz_bytes( + ForkName::Deneb => Ok(Some(RPCResponse::BlocksByRange(Arc::new( + SignedBeaconBlock::Deneb(SignedBeaconBlockDeneb::from_ssz_bytes( decoded_buffer, )?), )))), @@ -687,8 +687,8 @@ fn handle_v2_response( decoded_buffer, )?), )))), - ForkName::Eip4844 => Ok(Some(RPCResponse::BlocksByRoot(Arc::new( - SignedBeaconBlock::Eip4844(SignedBeaconBlockEip4844::from_ssz_bytes( + ForkName::Deneb => Ok(Some(RPCResponse::BlocksByRoot(Arc::new( + SignedBeaconBlock::Deneb(SignedBeaconBlockDeneb::from_ssz_bytes( decoded_buffer, )?), )))), @@ -753,19 +753,19 @@ mod tests { let altair_fork_epoch = Epoch::new(1); let merge_fork_epoch = Epoch::new(2); let capella_fork_epoch = Epoch::new(3); - let eip4844_fork_epoch = Epoch::new(4); + let deneb_fork_epoch = Epoch::new(4); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); chain_spec.bellatrix_fork_epoch = Some(merge_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); - chain_spec.eip4844_fork_epoch = Some(eip4844_fork_epoch); + chain_spec.deneb_fork_epoch = Some(deneb_fork_epoch); let current_slot = match fork_name { ForkName::Base => Slot::new(0), ForkName::Altair => altair_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Merge => merge_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Capella => capella_fork_epoch.start_slot(Spec::slots_per_epoch()), - ForkName::Eip4844 => eip4844_fork_epoch.start_slot(Spec::slots_per_epoch()), + ForkName::Deneb => deneb_fork_epoch.start_slot(Spec::slots_per_epoch()), }; ForkContext::new::(current_slot, Hash256::zero(), &chain_spec) } diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index bf67c943598..d627ae9f6e6 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -83,8 +83,8 @@ lazy_static! { + types::ExecutionPayload::::max_execution_payload_capella_size() // adding max size of execution payload (~16gb) + ssz::BYTES_PER_LENGTH_OFFSET; // Adding the additional ssz offset for the `ExecutionPayload` field - pub static ref SIGNED_BEACON_BLOCK_EIP4844_MAX: usize = *SIGNED_BEACON_BLOCK_CAPELLA_MAX_WITHOUT_PAYLOAD - + types::ExecutionPayload::::max_execution_payload_eip4844_size() // adding max size of execution payload (~16gb) + pub static ref SIGNED_BEACON_BLOCK_DENEB_MAX: usize = *SIGNED_BEACON_BLOCK_CAPELLA_MAX_WITHOUT_PAYLOAD + + types::ExecutionPayload::::max_execution_payload_deneb_size() // adding max size of execution payload (~16gb) + ssz::BYTES_PER_LENGTH_OFFSET // Adding the additional offsets for the `ExecutionPayload` + (::ssz_fixed_len() * ::max_blobs_per_block()) + ssz::BYTES_PER_LENGTH_OFFSET; // Length offset for the blob commitments field. @@ -119,7 +119,7 @@ lazy_static! { //FIXME(sean) these are underestimates pub static ref SIGNED_BLOCK_AND_BLOBS_MIN: usize = *BLOB_SIDECAR_MIN + *SIGNED_BEACON_BLOCK_BASE_MIN; - pub static ref SIGNED_BLOCK_AND_BLOBS_MAX: usize =*BLOB_SIDECAR_MAX + *SIGNED_BEACON_BLOCK_EIP4844_MAX; + pub static ref SIGNED_BLOCK_AND_BLOBS_MAX: usize =*BLOB_SIDECAR_MAX + *SIGNED_BEACON_BLOCK_DENEB_MAX; } /// The maximum bytes that can be sent across the RPC pre-merge. @@ -128,7 +128,7 @@ pub(crate) const MAX_RPC_SIZE: usize = 1_048_576; // 1M pub(crate) const MAX_RPC_SIZE_POST_MERGE: usize = 10 * 1_048_576; // 10M pub(crate) const MAX_RPC_SIZE_POST_CAPELLA: usize = 10 * 1_048_576; // 10M // FIXME(sean) should this be increased to account for blobs? -pub(crate) const MAX_RPC_SIZE_POST_EIP4844: usize = 10 * 1_048_576; // 10M +pub(crate) const MAX_RPC_SIZE_POST_DENEB: usize = 10 * 1_048_576; // 10M /// The protocol prefix the RPC protocol id. const PROTOCOL_PREFIX: &str = "/eth2/beacon_chain/req"; /// Time allowed for the first byte of a request to arrive before we time out (Time To First Byte). @@ -143,7 +143,7 @@ pub fn max_rpc_size(fork_context: &ForkContext) -> usize { ForkName::Altair | ForkName::Base => MAX_RPC_SIZE, ForkName::Merge => MAX_RPC_SIZE_POST_MERGE, ForkName::Capella => MAX_RPC_SIZE_POST_CAPELLA, - ForkName::Eip4844 => MAX_RPC_SIZE_POST_EIP4844, + ForkName::Deneb => MAX_RPC_SIZE_POST_DENEB, } } @@ -168,9 +168,9 @@ pub fn rpc_block_limits_by_fork(current_fork: ForkName) -> RpcLimits { *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks *SIGNED_BEACON_BLOCK_CAPELLA_MAX, // Capella block is larger than base, altair and merge blocks ), - ForkName::Eip4844 => RpcLimits::new( + ForkName::Deneb => RpcLimits::new( *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks - *SIGNED_BEACON_BLOCK_EIP4844_MAX, // EIP 4844 block is larger than all prior fork blocks + *SIGNED_BEACON_BLOCK_DENEB_MAX, // EIP 4844 block is larger than all prior fork blocks ), } } @@ -282,7 +282,7 @@ impl UpgradeInfo for RPCProtocol { ProtocolId::new(Protocol::MetaData, Version::V1, Encoding::SSZSnappy), ]; - if let ForkName::Eip4844 = self.fork_context.current_fork() { + if let ForkName::Deneb = self.fork_context.current_fork() { supported_protocols.extend_from_slice(&[ ProtocolId::new(Protocol::BlobsByRoot, Version::V1, Encoding::SSZSnappy), ProtocolId::new(Protocol::BlobsByRange, Version::V1, Encoding::SSZSnappy), diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 87012859afa..49cd8860b33 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -184,9 +184,9 @@ impl PubsubMessage { SignedBeaconBlockMerge::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, ), - Some(ForkName::Eip4844) => { + Some(ForkName::Deneb) => { return Err( - "beacon_block topic is not used from eip4844 fork onwards" + "beacon_block topic is not used from deneb fork onwards" .to_string(), ) } @@ -205,7 +205,7 @@ impl PubsubMessage { } GossipKind::BlobSidecar(blob_index) => { match fork_context.from_context_bytes(gossip_topic.fork_digest) { - Some(ForkName::Eip4844) => { + Some(ForkName::Deneb) => { let blob_sidecar = SignedBlobSidecar::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?; Ok(PubsubMessage::BlobSidecar(Box::new(( diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index d9deaaf0510..3a35c070250 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -47,7 +47,7 @@ pub fn fork_core_topics(fork_name: &ForkName) -> Vec { ForkName::Altair => ALTAIR_CORE_TOPICS.to_vec(), ForkName::Merge => vec![], ForkName::Capella => CAPELLA_CORE_TOPICS.to_vec(), - ForkName::Eip4844 => vec![], // TODO + ForkName::Deneb => vec![], // TODO } } diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index bd153284edb..40cc77b8b89 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -26,19 +26,19 @@ pub fn fork_context(fork_name: ForkName) -> ForkContext { let altair_fork_epoch = Epoch::new(1); let merge_fork_epoch = Epoch::new(2); let capella_fork_epoch = Epoch::new(3); - let eip4844_fork_epoch = Epoch::new(4); + let deneb_fork_epoch = Epoch::new(4); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); chain_spec.bellatrix_fork_epoch = Some(merge_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); - chain_spec.eip4844_fork_epoch = Some(eip4844_fork_epoch); + chain_spec.deneb_fork_epoch = Some(deneb_fork_epoch); let current_slot = match fork_name { ForkName::Base => Slot::new(0), ForkName::Altair => altair_fork_epoch.start_slot(E::slots_per_epoch()), ForkName::Merge => merge_fork_epoch.start_slot(E::slots_per_epoch()), ForkName::Capella => capella_fork_epoch.start_slot(E::slots_per_epoch()), - ForkName::Eip4844 => eip4844_fork_epoch.start_slot(E::slots_per_epoch()), + ForkName::Deneb => deneb_fork_epoch.start_slot(E::slots_per_epoch()), }; ForkContext::new::(current_slot, Hash256::zero(), &chain_spec) } diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index ecb2eb424ba..e00834872f9 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -287,7 +287,7 @@ impl Worker { Err(BeaconChainError::NoKzgCommitmentsFieldOnBlock) => { debug!( self.log, - "Peer requested blobs for a pre-eip4844 block"; + "Peer requested blobs for a pre-deneb block"; "peer" => %peer_id, "block_root" => ?root, ); @@ -707,11 +707,11 @@ impl Worker { let data_availability_boundary = match self.chain.data_availability_boundary() { Some(boundary) => boundary, None => { - debug!(self.log, "Eip4844 fork is disabled"); + debug!(self.log, "Deneb fork is disabled"); self.send_error_response( peer_id, RPCResponseErrorCode::ServerError, - "Eip4844 fork is disabled".into(), + "Deneb fork is disabled".into(), request_id, ); return; diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index 9bbd4790d97..d7eca40fc27 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -382,7 +382,7 @@ fn test_parent_lookup_rpc_failure() { &mut cx, RPCError::ErrorResponse( RPCResponseErrorCode::ResourceUnavailable, - "older than eip4844".into(), + "older than deneb".into(), ), ); let id2 = rig.expect_parent_request(); @@ -424,7 +424,7 @@ fn test_parent_lookup_too_many_attempts() { &mut cx, RPCError::ErrorResponse( RPCResponseErrorCode::ResourceUnavailable, - "older than eip4844".into(), + "older than deneb".into(), ), ); } @@ -467,7 +467,7 @@ fn test_parent_lookup_too_many_download_attempts_no_blacklist() { &mut cx, RPCError::ErrorResponse( RPCResponseErrorCode::ResourceUnavailable, - "older than eip4844".into(), + "older than deneb".into(), ), ); } else { @@ -509,7 +509,7 @@ fn test_parent_lookup_too_many_processing_attempts_must_blacklist() { &mut cx, RPCError::ErrorResponse( RPCResponseErrorCode::ResourceUnavailable, - "older than eip4844".into(), + "older than deneb".into(), ), ); } diff --git a/beacon_node/network/src/sync/range_sync/batch.rs b/beacon_node/network/src/sync/range_sync/batch.rs index dda22dcfa7e..601ac6a18f1 100644 --- a/beacon_node/network/src/sync/range_sync/batch.rs +++ b/beacon_node/network/src/sync/range_sync/batch.rs @@ -151,10 +151,10 @@ impl BatchInfo { /// ... | 30 | 31 | 32 | 33 | 34 | ... | 61 | 62 | 63 | 64 | 65 | /// Batch 1 | Batch 2 | Batch 3 /// - /// NOTE: Removed the shift by one for eip4844 because otherwise the last batch before the blob + /// NOTE: Removed the shift by one for deneb because otherwise the last batch before the blob /// fork boundary will be of mixed type (all blocks and one last blockblob), and I don't want to /// deal with this for now. - /// This means finalization might be slower in eip4844 + /// This means finalization might be slower in deneb pub fn new(start_epoch: &Epoch, num_of_epochs: u64, batch_type: ByRangeRequestType) -> Self { let start_slot = start_epoch.start_slot(T::slots_per_epoch()); let end_slot = start_slot + num_of_epochs * T::slots_per_epoch(); diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index fc902d866f5..520a0c8d656 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -39,7 +39,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use types::blob_sidecar::BlobSidecarList; -use types::consts::eip4844::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; +use types::consts::deneb::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; use types::*; /// On-disk database that stores finalized states efficiently. @@ -1854,10 +1854,10 @@ impl, Cold: ItemStore> HotColdDB /// Try to prune blobs, approximating the current epoch from lower epoch numbers end (older /// end) and is useful when the data availability boundary is not at hand. pub fn try_prune_most_blobs(&self, force: bool) -> Result<(), Error> { - let eip4844_fork = match self.spec.eip4844_fork_epoch { + let deneb_fork = match self.spec.deneb_fork_epoch { Some(epoch) => epoch, None => { - debug!(self.log, "Eip4844 fork is disabled"); + debug!(self.log, "Deneb fork is disabled"); return Ok(()); } }; @@ -1865,7 +1865,7 @@ impl, Cold: ItemStore> HotColdDB // `split.slot` is not updated and current_epoch > split_epoch + 2. let min_current_epoch = self.get_split_slot().epoch(E::slots_per_epoch()) + Epoch::new(2); let min_data_availability_boundary = std::cmp::max( - eip4844_fork, + deneb_fork, min_current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS), ); @@ -1878,11 +1878,11 @@ impl, Cold: ItemStore> HotColdDB force: bool, data_availability_boundary: Option, ) -> Result<(), Error> { - let (data_availability_boundary, eip4844_fork) = - match (data_availability_boundary, self.spec.eip4844_fork_epoch) { + let (data_availability_boundary, deneb_fork) = + match (data_availability_boundary, self.spec.deneb_fork_epoch) { (Some(boundary_epoch), Some(fork_epoch)) => (boundary_epoch, fork_epoch), _ => { - debug!(self.log, "Eip4844 fork is disabled"); + debug!(self.log, "Deneb fork is disabled"); return Ok(()); } }; @@ -1900,7 +1900,7 @@ impl, Cold: ItemStore> HotColdDB let blob_info = self.get_blob_info(); let oldest_blob_slot = blob_info .oldest_blob_slot - .unwrap_or_else(|| eip4844_fork.start_slot(E::slots_per_epoch())); + .unwrap_or_else(|| deneb_fork.start_slot(E::slots_per_epoch())); // The last entirely pruned epoch, blobs sidecar pruning may have stopped early in the // middle of an epoch otherwise the oldest blob slot is a start slot. diff --git a/beacon_node/store/src/impls/execution_payload.rs b/beacon_node/store/src/impls/execution_payload.rs index f4e6c1ea661..6445dad3886 100644 --- a/beacon_node/store/src/impls/execution_payload.rs +++ b/beacon_node/store/src/impls/execution_payload.rs @@ -1,7 +1,7 @@ use crate::{DBColumn, Error, StoreItem}; use ssz::{Decode, Encode}; use types::{ - BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, + BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, }; @@ -24,7 +24,7 @@ macro_rules! impl_store_item { } impl_store_item!(ExecutionPayloadMerge); impl_store_item!(ExecutionPayloadCapella); -impl_store_item!(ExecutionPayloadEip4844); +impl_store_item!(ExecutionPayloadDeneb); impl_store_item!(BlobSidecarList); /// This fork-agnostic implementation should be only used for writing. @@ -41,8 +41,8 @@ impl StoreItem for ExecutionPayload { } fn from_store_bytes(bytes: &[u8]) -> Result { - ExecutionPayloadEip4844::from_ssz_bytes(bytes) - .map(Self::Eip4844) + ExecutionPayloadDeneb::from_ssz_bytes(bytes) + .map(Self::Deneb) .or_else(|_| { ExecutionPayloadCapella::from_ssz_bytes(bytes) .map(Self::Capella) diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index 55697bd3160..5f864c44a3d 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -15,7 +15,7 @@ use types::*; /// /// Utilises lazy-loading from separate storage for its vector fields. #[superstruct( - variants(Base, Altair, Merge, Capella, Eip4844), + variants(Base, Altair, Merge, Capella, Deneb), variant_attributes(derive(Debug, PartialEq, Clone, Encode, Decode)) )] #[derive(Debug, PartialEq, Clone, Encode)] @@ -67,9 +67,9 @@ where pub current_epoch_attestations: VariableList, T::MaxPendingAttestations>, // Participation (Altair and later) - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub previous_epoch_participation: VariableList, - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub current_epoch_participation: VariableList, // Finality @@ -79,13 +79,13 @@ where pub finalized_checkpoint: Checkpoint, // Inactivity - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub inactivity_scores: VariableList, // Light-client sync committees - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub current_sync_committee: Arc>, - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub next_sync_committee: Arc>, // Execution @@ -100,19 +100,19 @@ where )] pub latest_execution_payload_header: ExecutionPayloadHeaderCapella, #[superstruct( - only(Eip4844), - partial_getter(rename = "latest_execution_payload_header_eip4844") + only(Deneb), + partial_getter(rename = "latest_execution_payload_header_deneb") )] - pub latest_execution_payload_header: ExecutionPayloadHeaderEip4844, + pub latest_execution_payload_header: ExecutionPayloadHeaderDeneb, // Capella - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Deneb))] pub next_withdrawal_index: u64, - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Deneb))] pub next_withdrawal_validator_index: u64, #[ssz(skip_serializing, skip_deserializing)] - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Deneb))] pub historical_summaries: Option>, } @@ -227,11 +227,11 @@ impl PartialBeaconState { ], [historical_summaries] ), - BeaconState::Eip4844(s) => impl_from_state_forgetful!( + BeaconState::Deneb(s) => impl_from_state_forgetful!( s, outer, - Eip4844, - PartialBeaconStateEip4844, + Deneb, + PartialBeaconStateDeneb, [ previous_epoch_participation, current_epoch_participation, @@ -472,10 +472,10 @@ impl TryInto> for PartialBeaconState { ], [historical_summaries] ), - PartialBeaconState::Eip4844(inner) => impl_try_into_beacon_state!( + PartialBeaconState::Deneb(inner) => impl_try_into_beacon_state!( inner, - Eip4844, - BeaconStateEip4844, + Deneb, + BeaconStateDeneb, [ previous_epoch_participation, current_epoch_participation, diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 543b3fda668..b8a9c9bcbe7 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -940,7 +940,7 @@ impl ForkVersionDeserialize for SsePayloadAttributes { ForkName::Merge => serde_json::from_value(value) .map(Self::V1) .map_err(serde::de::Error::custom), - ForkName::Capella | ForkName::Eip4844 => serde_json::from_value(value) + ForkName::Capella | ForkName::Deneb => serde_json::from_value(value) .map(Self::V2) .map_err(serde::de::Error::custom), ForkName::Base | ForkName::Altair => Err(serde::de::Error::custom(format!( @@ -1306,7 +1306,7 @@ impl> ForkVersionDeserialize D, >(value, fork_name)?)) } - ForkName::Eip4844 => Ok(BlockContents::BlockAndBlobSidecars( + ForkName::Deneb => Ok(BlockContents::BlockAndBlobSidecars( BeaconBlockAndBlobSidecars::deserialize_by_fork::<'de, D>(value, fork_name)?, )), } @@ -1369,7 +1369,7 @@ impl> From SignedBlockContents::Block(block), //TODO: error handling, this should be try from - SignedBeaconBlock::Eip4844(_block) => todo!(), + SignedBeaconBlock::Deneb(_block) => todo!(), } } } diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml b/common/eth2_network_config/built_in_network_configs/deneb/boot_enr.yaml similarity index 100% rename from common/eth2_network_config/built_in_network_configs/eip4844/boot_enr.yaml rename to common/eth2_network_config/built_in_network_configs/deneb/boot_enr.yaml diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml b/common/eth2_network_config/built_in_network_configs/deneb/config.yaml similarity index 89% rename from common/eth2_network_config/built_in_network_configs/eip4844/config.yaml rename to common/eth2_network_config/built_in_network_configs/deneb/config.yaml index f7334b6187c..72d3fd977dd 100644 --- a/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/deneb/config.yaml @@ -1,6 +1,6 @@ # Extends the mainnet preset PRESET_BASE: 'mainnet' -CONFIG_NAME: 'eip4844' # needs to exist because of Prysm. Otherwise it conflicts with mainnet genesis and needs to match configuration in common_eth2_config/src/lib.rs to pass lh ci. +CONFIG_NAME: 'deneb' # needs to exist because of Prysm. Otherwise it conflicts with mainnet genesis and needs to match configuration in common_eth2_config/src/lib.rs to pass lh ci. # Genesis # --------------------------------------------------------------- @@ -33,10 +33,10 @@ TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 CAPELLA_FORK_VERSION: 0x40484404 CAPELLA_FORK_EPOCH: 1 -# EIP4844/Deneb +# DENEB/Deneb # TODO: Rename to Deneb once specs/clients support it -EIP4844_FORK_VERSION: 0x50484404 -EIP4844_FORK_EPOCH: 5 +DENEB_FORK_VERSION: 0x50484404 +DENEB_FORK_EPOCH: 5 # Time parameters # --------------------------------------------------------------- diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/genesis.ssz.zip b/common/eth2_network_config/built_in_network_configs/deneb/genesis.ssz.zip similarity index 100% rename from common/eth2_network_config/built_in_network_configs/eip4844/genesis.ssz.zip rename to common/eth2_network_config/built_in_network_configs/deneb/genesis.ssz.zip diff --git a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml index 6aa2c9590a5..76f81837aa5 100644 --- a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml @@ -39,9 +39,9 @@ BELLATRIX_FORK_EPOCH: 385536 # Capella CAPELLA_FORK_VERSION: 0x03000064 CAPELLA_FORK_EPOCH: 18446744073709551615 -# Eip4844 -EIP4844_FORK_VERSION: 0x04000064 -EIP4844_FORK_EPOCH: 18446744073709551615 +# Deneb +DENEB_FORK_VERSION: 0x04000064 +DENEB_FORK_EPOCH: 18446744073709551615 # Sharding SHARDING_FORK_VERSION: 0x03000064 SHARDING_FORK_EPOCH: 18446744073709551615 diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml index b5ce752b700..9ee66394c63 100644 --- a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml @@ -39,9 +39,9 @@ BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC # Capella CAPELLA_FORK_VERSION: 0x03000000 CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC -# Eip4844 -EIP4844_FORK_VERSION: 0x04000000 -EIP4844_FORK_EPOCH: 18446744073709551615 +# Deneb +DENEB_FORK_VERSION: 0x04000000 +DENEB_FORK_EPOCH: 18446744073709551615 # Sharding SHARDING_FORK_VERSION: 0x03000000 SHARDING_FORK_EPOCH: 18446744073709551615 diff --git a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml index 4ba006ec945..b7ebb68f5c0 100644 --- a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml @@ -32,9 +32,9 @@ TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 CAPELLA_FORK_VERSION: 0x90000072 CAPELLA_FORK_EPOCH: 56832 -# Eip4844 -EIP4844_FORK_VERSION: 0x03001020 -EIP4844_FORK_EPOCH: 18446744073709551615 +# Deneb +DENEB_FORK_VERSION: 0x03001020 +DENEB_FORK_EPOCH: 18446744073709551615 # Sharding SHARDING_FORK_VERSION: 0x04001020 diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 3221a5cd6e2..0d434ba741f 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -70,8 +70,8 @@ impl Eth2NetworkConfig { fn from_hardcoded_net(net: &HardcodedNet) -> Result { let config: Config = serde_yaml::from_reader(net.config) .map_err(|e| format!("Unable to parse yaml config: {:?}", e))?; - let kzg_trusted_setup = if let Some(epoch) = config.eip4844_fork_epoch { - // Only load the trusted setup if the eip4844 fork epoch is set + let kzg_trusted_setup = if let Some(epoch) = config.deneb_fork_epoch { + // Only load the trusted setup if the deneb fork epoch is set if epoch.value != Epoch::max_value() { let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP) .map_err(|e| format!("Unable to read trusted setup file: {}", e))?; @@ -236,8 +236,8 @@ impl Eth2NetworkConfig { None }; - let kzg_trusted_setup = if let Some(epoch) = config.eip4844_fork_epoch { - // Only load the trusted setup if the eip4844 fork epoch is set + let kzg_trusted_setup = if let Some(epoch) = config.deneb_fork_epoch { + // Only load the trusted setup if the deneb fork epoch is set if epoch.value != Epoch::max_value() { let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP) .map_err(|e| format!("Unable to read trusted setup file: {}", e))?; diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index d717c31aaf8..96c520f7582 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -751,9 +751,9 @@ where (parent_justified, parent_finalized) } else { let justification_and_finalization_state = match block { - // TODO(eip4844): Ensure that the final specification + // TODO(deneb): Ensure that the final specification // does not substantially modify per epoch processing. - BeaconBlockRef::Eip4844(_) + BeaconBlockRef::Deneb(_) | BeaconBlockRef::Capella(_) | BeaconBlockRef::Merge(_) | BeaconBlockRef::Altair(_) => { diff --git a/consensus/state_processing/src/common/slash_validator.rs b/consensus/state_processing/src/common/slash_validator.rs index 77cd1a32659..94148ff6cfc 100644 --- a/consensus/state_processing/src/common/slash_validator.rs +++ b/consensus/state_processing/src/common/slash_validator.rs @@ -53,7 +53,7 @@ pub fn slash_validator( BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => whistleblower_reward + | BeaconState::Deneb(_) => whistleblower_reward .safe_mul(PROPOSER_WEIGHT)? .safe_div(WEIGHT_DENOMINATOR)?, }; diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 3f9328f4d5c..244cdc588b3 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -3,7 +3,7 @@ use super::per_block_processing::{ }; use crate::common::DepositDataTree; use crate::upgrade::{ - upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_eip4844, + upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, }; use safe_arith::{ArithError, SafeArith}; use tree_hash::TreeHash; @@ -93,20 +93,20 @@ pub fn initialize_beacon_state_from_eth1( } } - // Upgrade to eip4844 if configured from genesis + // Upgrade to deneb if configured from genesis if spec - .eip4844_fork_epoch + .deneb_fork_epoch .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) { - upgrade_to_eip4844(&mut state, spec)?; + upgrade_to_deneb(&mut state, spec)?; // Remove intermediate Capella fork from `state.fork`. - state.fork_mut().previous_version = spec.eip4844_fork_version; + state.fork_mut().previous_version = spec.deneb_fork_version; // Override latest execution payload header. - // See https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#testing - if let Some(ExecutionPayloadHeader::Eip4844(header)) = execution_payload_header { - *state.latest_execution_payload_header_eip4844_mut()? = header; + // See https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/beacon-chain.md#testing + if let Some(ExecutionPayloadHeader::Deneb(header)) = execution_payload_header { + *state.latest_execution_payload_header_deneb_mut()? = header; } } diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index b2d7e000723..d81665dbc74 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -13,7 +13,7 @@ pub use self::verify_attester_slashing::{ pub use self::verify_proposer_slashing::verify_proposer_slashing; pub use altair::sync_committee::process_sync_aggregate; pub use block_signature_verifier::{BlockSignatureVerifier, ParallelSignatureSets}; -pub use eip4844::eip4844::process_blob_kzg_commitments; +pub use deneb::deneb::process_blob_kzg_commitments; pub use is_valid_indexed_attestation::is_valid_indexed_attestation; pub use process_operations::process_operations; pub use verify_attestation::{ @@ -27,7 +27,7 @@ pub use verify_exit::verify_exit; pub mod altair; pub mod block_signature_verifier; -pub mod eip4844; +pub mod deneb; pub mod errors; mod is_valid_indexed_attestation; pub mod process_operations; @@ -405,9 +405,9 @@ pub fn process_execution_payload>( _ => return Err(BlockProcessingError::IncorrectStateType), } } - ExecutionPayloadHeaderRefMut::Eip4844(header_mut) => { + ExecutionPayloadHeaderRefMut::Deneb(header_mut) => { match payload.to_execution_payload_header() { - ExecutionPayloadHeader::Eip4844(header) => *header_mut = header, + ExecutionPayloadHeader::Deneb(header) => *header_mut = header, _ => return Err(BlockProcessingError::IncorrectStateType), } } @@ -523,7 +523,7 @@ pub fn process_withdrawals>( ) -> Result<(), BlockProcessingError> { match state { BeaconState::Merge(_) => Ok(()), - BeaconState::Capella(_) | BeaconState::Eip4844(_) => { + BeaconState::Capella(_) | BeaconState::Deneb(_) => { let expected_withdrawals = get_expected_withdrawals(state, spec)?; let expected_root = expected_withdrawals.tree_hash_root(); let withdrawals_root = payload.withdrawals_root()?; diff --git a/consensus/state_processing/src/per_block_processing/eip4844.rs b/consensus/state_processing/src/per_block_processing/deneb.rs similarity index 67% rename from consensus/state_processing/src/per_block_processing/eip4844.rs rename to consensus/state_processing/src/per_block_processing/deneb.rs index 23ab3c5c074..68d53da9d76 100644 --- a/consensus/state_processing/src/per_block_processing/eip4844.rs +++ b/consensus/state_processing/src/per_block_processing/deneb.rs @@ -1,2 +1,2 @@ #[allow(clippy::module_inception)] -pub mod eip4844; +pub mod deneb; diff --git a/consensus/state_processing/src/per_block_processing/eip4844/eip4844.rs b/consensus/state_processing/src/per_block_processing/deneb/deneb.rs similarity index 98% rename from consensus/state_processing/src/per_block_processing/eip4844/eip4844.rs rename to consensus/state_processing/src/per_block_processing/deneb/deneb.rs index 8696336123d..5935d6f2688 100644 --- a/consensus/state_processing/src/per_block_processing/eip4844/eip4844.rs +++ b/consensus/state_processing/src/per_block_processing/deneb/deneb.rs @@ -3,7 +3,7 @@ use eth2_hashing::hash_fixed; use itertools::{EitherOrBoth, Itertools}; use safe_arith::SafeArith; use ssz::Decode; -use types::consts::eip4844::{BLOB_TX_TYPE, VERSIONED_HASH_VERSION_KZG}; +use types::consts::deneb::{BLOB_TX_TYPE, VERSIONED_HASH_VERSION_KZG}; use types::{ AbstractExecPayload, BeaconBlockBodyRef, EthSpec, ExecPayload, KzgCommitment, Transaction, Transactions, VersionedHash, diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 8a6163f29b9..ed831953fe2 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -257,7 +257,7 @@ pub fn process_attestations>( BeaconBlockBodyRef::Altair(_) | BeaconBlockBodyRef::Merge(_) | BeaconBlockBodyRef::Capella(_) - | BeaconBlockBodyRef::Eip4844(_) => { + | BeaconBlockBodyRef::Deneb(_) => { altair::process_attestations( state, block_body.attestations(), diff --git a/consensus/state_processing/src/per_epoch_processing.rs b/consensus/state_processing/src/per_epoch_processing.rs index 996e39c27fb..d5d06037cd8 100644 --- a/consensus/state_processing/src/per_epoch_processing.rs +++ b/consensus/state_processing/src/per_epoch_processing.rs @@ -40,7 +40,7 @@ pub fn process_epoch( match state { BeaconState::Base(_) => base::process_epoch(state, spec), BeaconState::Altair(_) | BeaconState::Merge(_) => altair::process_epoch(state, spec), - BeaconState::Capella(_) | BeaconState::Eip4844(_) => capella::process_epoch(state, spec), + BeaconState::Capella(_) | BeaconState::Deneb(_) => capella::process_epoch(state, spec), } } diff --git a/consensus/state_processing/src/per_slot_processing.rs b/consensus/state_processing/src/per_slot_processing.rs index 8d2600bb41e..75b8c6b0061 100644 --- a/consensus/state_processing/src/per_slot_processing.rs +++ b/consensus/state_processing/src/per_slot_processing.rs @@ -1,5 +1,5 @@ use crate::upgrade::{ - upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_eip4844, + upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, }; use crate::{per_epoch_processing::EpochProcessingSummary, *}; use safe_arith::{ArithError, SafeArith}; @@ -61,9 +61,9 @@ pub fn per_slot_processing( if spec.capella_fork_epoch == Some(state.current_epoch()) { upgrade_to_capella(state, spec)?; } - // Eip4844 - if spec.eip4844_fork_epoch == Some(state.current_epoch()) { - upgrade_to_eip4844(state, spec)?; + // Deneb + if spec.deneb_fork_epoch == Some(state.current_epoch()) { + upgrade_to_deneb(state, spec)?; } } diff --git a/consensus/state_processing/src/upgrade.rs b/consensus/state_processing/src/upgrade.rs index 01b65710564..1509ee0e50f 100644 --- a/consensus/state_processing/src/upgrade.rs +++ b/consensus/state_processing/src/upgrade.rs @@ -1,9 +1,9 @@ pub mod altair; pub mod capella; -pub mod eip4844; +pub mod deneb; pub mod merge; pub use altair::upgrade_to_altair; pub use capella::upgrade_to_capella; -pub use eip4844::upgrade_to_eip4844; +pub use deneb::upgrade_to_deneb; pub use merge::upgrade_to_bellatrix; diff --git a/consensus/state_processing/src/upgrade/eip4844.rs b/consensus/state_processing/src/upgrade/deneb.rs similarity index 89% rename from consensus/state_processing/src/upgrade/eip4844.rs rename to consensus/state_processing/src/upgrade/deneb.rs index 4f6ff9d1943..76dbcd068a1 100644 --- a/consensus/state_processing/src/upgrade/eip4844.rs +++ b/consensus/state_processing/src/upgrade/deneb.rs @@ -1,8 +1,8 @@ use std::mem; -use types::{BeaconState, BeaconStateEip4844, BeaconStateError as Error, ChainSpec, EthSpec, Fork}; +use types::{BeaconState, BeaconStateDeneb, BeaconStateError as Error, ChainSpec, EthSpec, Fork}; -/// Transform a `Capella` state into an `Eip4844` state. -pub fn upgrade_to_eip4844( +/// Transform a `Capella` state into an `Deneb` state. +pub fn upgrade_to_deneb( pre_state: &mut BeaconState, spec: &ChainSpec, ) -> Result<(), Error> { @@ -16,14 +16,14 @@ pub fn upgrade_to_eip4844( // // Fixed size vectors get cloned because replacing them would require the same size // allocation as cloning. - let post = BeaconState::Eip4844(BeaconStateEip4844 { + let post = BeaconState::Deneb(BeaconStateDeneb { // Versioning genesis_time: pre.genesis_time, genesis_validators_root: pre.genesis_validators_root, slot: pre.slot, fork: Fork { previous_version: previous_fork_version, - current_version: spec.eip4844_fork_version, + current_version: spec.deneb_fork_version, epoch, }, // History @@ -56,7 +56,7 @@ pub fn upgrade_to_eip4844( current_sync_committee: pre.current_sync_committee.clone(), next_sync_committee: pre.next_sync_committee.clone(), // Execution - latest_execution_payload_header: pre.latest_execution_payload_header.upgrade_to_eip4844(), + latest_execution_payload_header: pre.latest_execution_payload_header.upgrade_to_deneb(), // Capella next_withdrawal_index: pre.next_withdrawal_index, next_withdrawal_validator_index: pre.next_withdrawal_validator_index, diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 0f26cd0e5e7..27f15c9ed07 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -1,5 +1,5 @@ use crate::beacon_block_body::{ - BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyEip4844, BeaconBlockBodyMerge, + BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyDeneb, BeaconBlockBodyMerge, BeaconBlockBodyRef, BeaconBlockBodyRefMut, }; use crate::test_utils::TestRandom; @@ -17,7 +17,7 @@ use tree_hash_derive::TreeHash; /// A block of the `BeaconChain`. #[superstruct( - variants(Base, Altair, Merge, Capella, Eip4844), + variants(Base, Altair, Merge, Capella, Deneb), variant_attributes( derive( Debug, @@ -72,8 +72,8 @@ pub struct BeaconBlock = FullPayload pub body: BeaconBlockBodyMerge, #[superstruct(only(Capella), partial_getter(rename = "body_capella"))] pub body: BeaconBlockBodyCapella, - #[superstruct(only(Eip4844), partial_getter(rename = "body_eip4844"))] - pub body: BeaconBlockBodyEip4844, + #[superstruct(only(Deneb), partial_getter(rename = "body_deneb"))] + pub body: BeaconBlockBodyDeneb, } pub type BlindedBeaconBlock = BeaconBlock>; @@ -126,8 +126,8 @@ impl> BeaconBlock { /// Usually it's better to prefer `from_ssz_bytes` which will decode the correct variant based /// on the fork slot. pub fn any_from_ssz_bytes(bytes: &[u8]) -> Result { - BeaconBlockEip4844::from_ssz_bytes(bytes) - .map(BeaconBlock::Eip4844) + BeaconBlockDeneb::from_ssz_bytes(bytes) + .map(BeaconBlock::Deneb) .or_else(|_| BeaconBlockCapella::from_ssz_bytes(bytes).map(BeaconBlock::Capella)) .or_else(|_| BeaconBlockMerge::from_ssz_bytes(bytes).map(BeaconBlock::Merge)) .or_else(|_| BeaconBlockAltair::from_ssz_bytes(bytes).map(BeaconBlock::Altair)) @@ -206,7 +206,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockRef<'a, T, Payl BeaconBlockRef::Altair { .. } => ForkName::Altair, BeaconBlockRef::Merge { .. } => ForkName::Merge, BeaconBlockRef::Capella { .. } => ForkName::Capella, - BeaconBlockRef::Eip4844 { .. } => ForkName::Eip4844, + BeaconBlockRef::Deneb { .. } => ForkName::Deneb, }; if fork_at_slot == object_fork { @@ -560,15 +560,15 @@ impl> EmptyBlock for BeaconBlockCape } } -impl> EmptyBlock for BeaconBlockEip4844 { - /// Returns an empty Eip4844 block to be used during genesis. +impl> EmptyBlock for BeaconBlockDeneb { + /// Returns an empty Deneb block to be used during genesis. fn empty(spec: &ChainSpec) -> Self { - BeaconBlockEip4844 { + BeaconBlockDeneb { slot: spec.genesis_slot, proposer_index: 0, parent_root: Hash256::zero(), state_root: Hash256::zero(), - body: BeaconBlockBodyEip4844 { + body: BeaconBlockBodyDeneb { randao_reveal: Signature::empty(), eth1_data: Eth1Data { deposit_root: Hash256::zero(), @@ -582,7 +582,7 @@ impl> EmptyBlock for BeaconBlockEip4 deposits: VariableList::empty(), voluntary_exits: VariableList::empty(), sync_aggregate: SyncAggregate::empty(), - execution_payload: Payload::Eip4844::default(), + execution_payload: Payload::Deneb::default(), bls_to_execution_changes: VariableList::empty(), blob_kzg_commitments: VariableList::empty(), }, @@ -669,7 +669,7 @@ impl_from!(BeaconBlockBase, >, >, |body: impl_from!(BeaconBlockAltair, >, >, |body: BeaconBlockBodyAltair<_, _>| body.into()); impl_from!(BeaconBlockMerge, >, >, |body: BeaconBlockBodyMerge<_, _>| body.into()); impl_from!(BeaconBlockCapella, >, >, |body: BeaconBlockBodyCapella<_, _>| body.into()); -impl_from!(BeaconBlockEip4844, >, >, |body: BeaconBlockBodyEip4844<_, _>| body.into()); +impl_from!(BeaconBlockDeneb, >, >, |body: BeaconBlockBodyDeneb<_, _>| body.into()); // We can clone blocks with payloads to blocks without payloads, without cloning the payload. macro_rules! impl_clone_as_blinded { @@ -701,7 +701,7 @@ impl_clone_as_blinded!(BeaconBlockBase, >, >, >); impl_clone_as_blinded!(BeaconBlockMerge, >, >); impl_clone_as_blinded!(BeaconBlockCapella, >, >); -impl_clone_as_blinded!(BeaconBlockEip4844, >, >); +impl_clone_as_blinded!(BeaconBlockDeneb, >, >); // A reference to a full beacon block can be cloned into a blinded beacon block, without cloning the // execution payload. @@ -820,16 +820,16 @@ mod tests { #[test] fn roundtrip_4844_block() { let rng = &mut XorShiftRng::from_seed([42; 16]); - let spec = &ForkName::Eip4844.make_genesis_spec(MainnetEthSpec::default_spec()); + let spec = &ForkName::Deneb.make_genesis_spec(MainnetEthSpec::default_spec()); - let inner_block = BeaconBlockEip4844 { + let inner_block = BeaconBlockDeneb { slot: Slot::random_for_test(rng), proposer_index: u64::random_for_test(rng), parent_root: Hash256::random_for_test(rng), state_root: Hash256::random_for_test(rng), - body: BeaconBlockBodyEip4844::random_for_test(rng), + body: BeaconBlockBodyDeneb::random_for_test(rng), }; - let block = BeaconBlock::Eip4844(inner_block.clone()); + let block = BeaconBlock::Deneb(inner_block.clone()); test_ssz_tree_hash_pair_with(&block, &inner_block, |bytes| { BeaconBlock::from_ssz_bytes(bytes, spec) @@ -851,12 +851,12 @@ mod tests { let altair_slot = altair_epoch.start_slot(E::slots_per_epoch()); let capella_epoch = altair_fork_epoch + 1; let capella_slot = capella_epoch.start_slot(E::slots_per_epoch()); - let eip4844_epoch = capella_epoch + 1; - let eip4844_slot = eip4844_epoch.start_slot(E::slots_per_epoch()); + let deneb_epoch = capella_epoch + 1; + let deneb_slot = deneb_epoch.start_slot(E::slots_per_epoch()); spec.altair_fork_epoch = Some(altair_epoch); spec.capella_fork_epoch = Some(capella_epoch); - spec.eip4844_fork_epoch = Some(eip4844_epoch); + spec.deneb_fork_epoch = Some(deneb_epoch); // BeaconBlockBase { @@ -924,10 +924,10 @@ mod tests { .expect_err("bad capella block cannot be decoded"); } - // BeaconBlockEip4844 + // BeaconBlockDeneb { - let good_block = BeaconBlock::Eip4844(BeaconBlockEip4844 { - slot: eip4844_slot, + let good_block = BeaconBlock::Deneb(BeaconBlockDeneb { + slot: deneb_slot, ..<_>::random_for_test(rng) }); // It's invalid to have an Capella block with a epoch lower than the fork epoch. @@ -939,11 +939,11 @@ mod tests { assert_eq!( BeaconBlock::from_ssz_bytes(&good_block.as_ssz_bytes(), &spec) - .expect("good eip4844 block can be decoded"), + .expect("good deneb block can be decoded"), good_block ); BeaconBlock::from_ssz_bytes(&bad_block.as_ssz_bytes(), &spec) - .expect_err("bad eip4844 block cannot be decoded"); + .expect_err("bad deneb block cannot be decoded"); } } } diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index e49f633459f..f2174467b2a 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -15,7 +15,7 @@ pub type KzgCommitments = VariableList::MaxBlob /// /// This *superstruct* abstracts over the hard-fork. #[superstruct( - variants(Base, Altair, Merge, Capella, Eip4844), + variants(Base, Altair, Merge, Capella, Deneb), variant_attributes( derive( Debug, @@ -53,7 +53,7 @@ pub struct BeaconBlockBody = FullPay pub attestations: VariableList, T::MaxAttestations>, pub deposits: VariableList, pub voluntary_exits: VariableList, - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub sync_aggregate: SyncAggregate, // We flatten the execution payload so that serde can use the name of the inner type, // either `execution_payload` for full payloads, or `execution_payload_header` for blinded @@ -64,13 +64,13 @@ pub struct BeaconBlockBody = FullPay #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] #[serde(flatten)] pub execution_payload: Payload::Capella, - #[superstruct(only(Eip4844), partial_getter(rename = "execution_payload_eip4844"))] + #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] #[serde(flatten)] - pub execution_payload: Payload::Eip4844, - #[superstruct(only(Capella, Eip4844))] + pub execution_payload: Payload::Deneb, + #[superstruct(only(Capella, Deneb))] pub bls_to_execution_changes: VariableList, - #[superstruct(only(Eip4844))] + #[superstruct(only(Deneb))] pub blob_kzg_commitments: KzgCommitments, #[superstruct(only(Base, Altair))] #[ssz(skip_serializing, skip_deserializing)] @@ -92,7 +92,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, T, Self::Base(_) | Self::Altair(_) => Err(Error::IncorrectStateVariant), Self::Merge(body) => Ok(Payload::Ref::from(&body.execution_payload)), Self::Capella(body) => Ok(Payload::Ref::from(&body.execution_payload)), - Self::Eip4844(body) => Ok(Payload::Ref::from(&body.execution_payload)), + Self::Deneb(body) => Ok(Payload::Ref::from(&body.execution_payload)), } } } @@ -105,7 +105,7 @@ impl<'a, T: EthSpec> BeaconBlockBodyRef<'a, T> { BeaconBlockBodyRef::Altair { .. } => ForkName::Altair, BeaconBlockBodyRef::Merge { .. } => ForkName::Merge, BeaconBlockBodyRef::Capella { .. } => ForkName::Capella, - BeaconBlockBodyRef::Eip4844 { .. } => ForkName::Eip4844, + BeaconBlockBodyRef::Deneb { .. } => ForkName::Deneb, } } } @@ -330,14 +330,14 @@ impl From>> } } -impl From>> +impl From>> for ( - BeaconBlockBodyEip4844>, - Option>, + BeaconBlockBodyDeneb>, + Option>, ) { - fn from(body: BeaconBlockBodyEip4844>) -> Self { - let BeaconBlockBodyEip4844 { + fn from(body: BeaconBlockBodyDeneb>) -> Self { + let BeaconBlockBodyDeneb { randao_reveal, eth1_data, graffiti, @@ -347,13 +347,13 @@ impl From>> deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayloadEip4844 { execution_payload }, + execution_payload: FullPayloadDeneb { execution_payload }, bls_to_execution_changes, blob_kzg_commitments, } = body; ( - BeaconBlockBodyEip4844 { + BeaconBlockBodyDeneb { randao_reveal, eth1_data, graffiti, @@ -363,7 +363,7 @@ impl From>> deposits, voluntary_exits, sync_aggregate, - execution_payload: BlindedPayloadEip4844 { + execution_payload: BlindedPayloadDeneb { execution_payload_header: From::from(&execution_payload), }, bls_to_execution_changes, @@ -455,9 +455,9 @@ impl BeaconBlockBodyCapella> { } } -impl BeaconBlockBodyEip4844> { - pub fn clone_as_blinded(&self) -> BeaconBlockBodyEip4844> { - let BeaconBlockBodyEip4844 { +impl BeaconBlockBodyDeneb> { + pub fn clone_as_blinded(&self) -> BeaconBlockBodyDeneb> { + let BeaconBlockBodyDeneb { randao_reveal, eth1_data, graffiti, @@ -467,12 +467,12 @@ impl BeaconBlockBodyEip4844> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayloadEip4844 { execution_payload }, + execution_payload: FullPayloadDeneb { execution_payload }, bls_to_execution_changes, blob_kzg_commitments, } = self; - BeaconBlockBodyEip4844 { + BeaconBlockBodyDeneb { randao_reveal: randao_reveal.clone(), eth1_data: eth1_data.clone(), graffiti: *graffiti, @@ -482,7 +482,7 @@ impl BeaconBlockBodyEip4844> { deposits: deposits.clone(), voluntary_exits: voluntary_exits.clone(), sync_aggregate: sync_aggregate.clone(), - execution_payload: BlindedPayloadEip4844 { + execution_payload: BlindedPayloadDeneb { execution_payload_header: execution_payload.into(), }, bls_to_execution_changes: bls_to_execution_changes.clone(), diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index c98df48d14e..d480c0fc32e 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -176,7 +176,7 @@ impl From for Hash256 { /// The state of the `BeaconChain` at some slot. #[superstruct( - variants(Base, Altair, Merge, Capella, Eip4844), + variants(Base, Altair, Merge, Capella, Deneb), variant_attributes( derive( Derivative, @@ -256,9 +256,9 @@ where pub current_epoch_attestations: VariableList, T::MaxPendingAttestations>, // Participation (Altair and later) - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub previous_epoch_participation: VariableList, - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub current_epoch_participation: VariableList, // Finality @@ -273,13 +273,13 @@ where // Inactivity #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub inactivity_scores: VariableList, // Light-client sync committees - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub current_sync_committee: Arc>, - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Deneb))] pub next_sync_committee: Arc>, // Execution @@ -294,20 +294,20 @@ where )] pub latest_execution_payload_header: ExecutionPayloadHeaderCapella, #[superstruct( - only(Eip4844), - partial_getter(rename = "latest_execution_payload_header_eip4844") + only(Deneb), + partial_getter(rename = "latest_execution_payload_header_deneb") )] - pub latest_execution_payload_header: ExecutionPayloadHeaderEip4844, + pub latest_execution_payload_header: ExecutionPayloadHeaderDeneb, // Capella - #[superstruct(only(Capella, Eip4844), partial_getter(copy))] + #[superstruct(only(Capella, Deneb), partial_getter(copy))] #[serde(with = "eth2_serde_utils::quoted_u64")] pub next_withdrawal_index: u64, - #[superstruct(only(Capella, Eip4844), partial_getter(copy))] + #[superstruct(only(Capella, Deneb), partial_getter(copy))] #[serde(with = "eth2_serde_utils::quoted_u64")] pub next_withdrawal_validator_index: u64, // Deep history valid from Capella onwards. - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Deneb))] pub historical_summaries: VariableList, // Caching (not in the spec) @@ -420,7 +420,7 @@ impl BeaconState { BeaconState::Altair { .. } => ForkName::Altair, BeaconState::Merge { .. } => ForkName::Merge, BeaconState::Capella { .. } => ForkName::Capella, - BeaconState::Eip4844 { .. } => ForkName::Eip4844, + BeaconState::Deneb { .. } => ForkName::Deneb, }; if fork_at_slot == object_fork { @@ -720,7 +720,7 @@ impl BeaconState { BeaconState::Capella(state) => Ok(ExecutionPayloadHeaderRef::Capella( &state.latest_execution_payload_header, )), - BeaconState::Eip4844(state) => Ok(ExecutionPayloadHeaderRef::Eip4844( + BeaconState::Deneb(state) => Ok(ExecutionPayloadHeaderRef::Deneb( &state.latest_execution_payload_header, )), } @@ -737,7 +737,7 @@ impl BeaconState { BeaconState::Capella(state) => Ok(ExecutionPayloadHeaderRefMut::Capella( &mut state.latest_execution_payload_header, )), - BeaconState::Eip4844(state) => Ok(ExecutionPayloadHeaderRefMut::Eip4844( + BeaconState::Deneb(state) => Ok(ExecutionPayloadHeaderRefMut::Deneb( &mut state.latest_execution_payload_header, )), } @@ -1168,7 +1168,7 @@ impl BeaconState { BeaconState::Altair(state) => (&mut state.validators, &mut state.balances), BeaconState::Merge(state) => (&mut state.validators, &mut state.balances), BeaconState::Capella(state) => (&mut state.validators, &mut state.balances), - BeaconState::Eip4844(state) => (&mut state.validators, &mut state.balances), + BeaconState::Deneb(state) => (&mut state.validators, &mut state.balances), } } @@ -1366,7 +1366,7 @@ impl BeaconState { BeaconState::Altair(state) => Ok(&mut state.current_epoch_participation), BeaconState::Merge(state) => Ok(&mut state.current_epoch_participation), BeaconState::Capella(state) => Ok(&mut state.current_epoch_participation), - BeaconState::Eip4844(state) => Ok(&mut state.current_epoch_participation), + BeaconState::Deneb(state) => Ok(&mut state.current_epoch_participation), } } else if epoch == self.previous_epoch() { match self { @@ -1374,7 +1374,7 @@ impl BeaconState { BeaconState::Altair(state) => Ok(&mut state.previous_epoch_participation), BeaconState::Merge(state) => Ok(&mut state.previous_epoch_participation), BeaconState::Capella(state) => Ok(&mut state.previous_epoch_participation), - BeaconState::Eip4844(state) => Ok(&mut state.previous_epoch_participation), + BeaconState::Deneb(state) => Ok(&mut state.previous_epoch_participation), } } else { Err(BeaconStateError::EpochOutOfBounds) @@ -1680,7 +1680,7 @@ impl BeaconState { BeaconState::Altair(inner) => BeaconState::Altair(inner.clone()), BeaconState::Merge(inner) => BeaconState::Merge(inner.clone()), BeaconState::Capella(inner) => BeaconState::Capella(inner.clone()), - BeaconState::Eip4844(inner) => BeaconState::Eip4844(inner.clone()), + BeaconState::Deneb(inner) => BeaconState::Deneb(inner.clone()), }; if config.committee_caches { *res.committee_caches_mut() = self.committee_caches().clone(); @@ -1849,7 +1849,7 @@ impl CompareFields for BeaconState { (BeaconState::Altair(x), BeaconState::Altair(y)) => x.compare_fields(y), (BeaconState::Merge(x), BeaconState::Merge(y)) => x.compare_fields(y), (BeaconState::Capella(x), BeaconState::Capella(y)) => x.compare_fields(y), - (BeaconState::Eip4844(x), BeaconState::Eip4844(y)) => x.compare_fields(y), + (BeaconState::Deneb(x), BeaconState::Deneb(y)) => x.compare_fields(y), _ => panic!("compare_fields: mismatched state variants",), } } diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 12f8afd9c73..ce6d7e0e611 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -11,7 +11,7 @@ use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; /// Container of the data that identifies an individual blob. -#[derive(Encode, Decode, Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Serialize, Deserialize, Encode, Decode, TreeHash, Clone, Debug, PartialEq, Eq, Hash)] pub struct BlobIdentifier { pub block_root: Hash256, pub index: u64, diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 487074c3383..1c07c9a021c 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -162,10 +162,10 @@ pub struct ChainSpec { pub max_validators_per_withdrawals_sweep: u64, /* - * Eip4844 hard fork params + * Deneb hard fork params */ - pub eip4844_fork_version: [u8; 4], - pub eip4844_fork_epoch: Option, + pub deneb_fork_version: [u8; 4], + pub deneb_fork_epoch: Option, /* * Networking @@ -255,8 +255,8 @@ impl ChainSpec { /// Returns the name of the fork which is active at `epoch`. pub fn fork_name_at_epoch(&self, epoch: Epoch) -> ForkName { - match self.eip4844_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Eip4844, + match self.deneb_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Deneb, _ => match self.capella_fork_epoch { Some(fork_epoch) if epoch >= fork_epoch => ForkName::Capella, _ => match self.bellatrix_fork_epoch { @@ -277,7 +277,7 @@ impl ChainSpec { ForkName::Altair => self.altair_fork_version, ForkName::Merge => self.bellatrix_fork_version, ForkName::Capella => self.capella_fork_version, - ForkName::Eip4844 => self.eip4844_fork_version, + ForkName::Deneb => self.deneb_fork_version, } } @@ -288,7 +288,7 @@ impl ChainSpec { ForkName::Altair => self.altair_fork_epoch, ForkName::Merge => self.bellatrix_fork_epoch, ForkName::Capella => self.capella_fork_epoch, - ForkName::Eip4844 => self.eip4844_fork_epoch, + ForkName::Deneb => self.deneb_fork_epoch, } } @@ -299,7 +299,7 @@ impl ChainSpec { BeaconState::Altair(_) => self.inactivity_penalty_quotient_altair, BeaconState::Merge(_) => self.inactivity_penalty_quotient_bellatrix, BeaconState::Capella(_) => self.inactivity_penalty_quotient_bellatrix, - BeaconState::Eip4844(_) => self.inactivity_penalty_quotient_bellatrix, + BeaconState::Deneb(_) => self.inactivity_penalty_quotient_bellatrix, } } @@ -313,7 +313,7 @@ impl ChainSpec { BeaconState::Altair(_) => self.proportional_slashing_multiplier_altair, BeaconState::Merge(_) => self.proportional_slashing_multiplier_bellatrix, BeaconState::Capella(_) => self.proportional_slashing_multiplier_bellatrix, - BeaconState::Eip4844(_) => self.proportional_slashing_multiplier_bellatrix, + BeaconState::Deneb(_) => self.proportional_slashing_multiplier_bellatrix, } } @@ -327,7 +327,7 @@ impl ChainSpec { BeaconState::Altair(_) => self.min_slashing_penalty_quotient_altair, BeaconState::Merge(_) => self.min_slashing_penalty_quotient_bellatrix, BeaconState::Capella(_) => self.min_slashing_penalty_quotient_bellatrix, - BeaconState::Eip4844(_) => self.min_slashing_penalty_quotient_bellatrix, + BeaconState::Deneb(_) => self.min_slashing_penalty_quotient_bellatrix, } } @@ -637,10 +637,10 @@ impl ChainSpec { max_validators_per_withdrawals_sweep: 16384, /* - * Eip4844 hard fork params + * Deneb hard fork params */ - eip4844_fork_version: [0x04, 0x00, 0x00, 0x00], - eip4844_fork_epoch: None, + deneb_fork_version: [0x04, 0x00, 0x00, 0x00], + deneb_fork_epoch: None, /* * Network specific @@ -709,9 +709,9 @@ impl ChainSpec { capella_fork_version: [0x03, 0x00, 0x00, 0x01], capella_fork_epoch: None, max_validators_per_withdrawals_sweep: 16, - // Eip4844 - eip4844_fork_version: [0x04, 0x00, 0x00, 0x01], - eip4844_fork_epoch: None, + // Deneb + deneb_fork_version: [0x04, 0x00, 0x00, 0x01], + deneb_fork_epoch: None, // Other network_id: 2, // lighthouse testnet network id deposit_chain_id: 5, @@ -874,10 +874,10 @@ impl ChainSpec { max_validators_per_withdrawals_sweep: 16384, /* - * Eip4844 hard fork params + * Deneb hard fork params */ - eip4844_fork_version: [0x04, 0x00, 0x00, 0x64], - eip4844_fork_epoch: None, + deneb_fork_version: [0x04, 0x00, 0x00, 0x64], + deneb_fork_epoch: None, /* * Network specific @@ -970,13 +970,13 @@ pub struct Config { #[serde(deserialize_with = "deserialize_fork_epoch")] pub capella_fork_epoch: Option>, - #[serde(default = "default_eip4844_fork_version")] + #[serde(default = "default_deneb_fork_version")] #[serde(with = "eth2_serde_utils::bytes_4_hex")] - eip4844_fork_version: [u8; 4], + deneb_fork_version: [u8; 4], #[serde(default)] #[serde(serialize_with = "serialize_fork_epoch")] #[serde(deserialize_with = "deserialize_fork_epoch")] - pub eip4844_fork_epoch: Option>, + pub deneb_fork_epoch: Option>, #[serde(with = "eth2_serde_utils::quoted_u64")] seconds_per_slot: u64, @@ -1020,7 +1020,7 @@ fn default_capella_fork_version() -> [u8; 4] { [0xff, 0xff, 0xff, 0xff] } -fn default_eip4844_fork_version() -> [u8; 4] { +fn default_deneb_fork_version() -> [u8; 4] { // This value shouldn't be used. [0xff, 0xff, 0xff, 0xff] } @@ -1125,9 +1125,9 @@ impl Config { capella_fork_epoch: spec .capella_fork_epoch .map(|epoch| MaybeQuoted { value: epoch }), - eip4844_fork_version: spec.eip4844_fork_version, - eip4844_fork_epoch: spec - .eip4844_fork_epoch + deneb_fork_version: spec.deneb_fork_version, + deneb_fork_epoch: spec + .deneb_fork_epoch .map(|epoch| MaybeQuoted { value: epoch }), seconds_per_slot: spec.seconds_per_slot, @@ -1176,8 +1176,8 @@ impl Config { bellatrix_fork_version, capella_fork_epoch, capella_fork_version, - eip4844_fork_epoch, - eip4844_fork_version, + deneb_fork_epoch, + deneb_fork_version, seconds_per_slot, seconds_per_eth1_block, min_validator_withdrawability_delay, @@ -1210,8 +1210,8 @@ impl Config { bellatrix_fork_version, capella_fork_epoch: capella_fork_epoch.map(|q| q.value), capella_fork_version, - eip4844_fork_epoch: eip4844_fork_epoch.map(|q| q.value), - eip4844_fork_version, + deneb_fork_epoch: deneb_fork_epoch.map(|q| q.value), + deneb_fork_version, seconds_per_slot, seconds_per_eth1_block, min_validator_withdrawability_delay, diff --git a/consensus/types/src/consts.rs b/consensus/types/src/consts.rs index a335cbd7b29..8d296ad942f 100644 --- a/consensus/types/src/consts.rs +++ b/consensus/types/src/consts.rs @@ -22,7 +22,7 @@ pub mod altair { pub mod merge { pub const INTERVALS_PER_SLOT: u64 = 3; } -pub mod eip4844 { +pub mod deneb { use crate::{Epoch, Uint256}; use lazy_static::lazy_static; diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 7a78dd5800c..03b767a17bd 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -103,7 +103,7 @@ pub trait EthSpec: type MaxBlsToExecutionChanges: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxWithdrawalsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* - * New in Eip4844 + * New in Deneb */ type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq; type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 7762afe9184..823483b0184 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -15,7 +15,7 @@ pub type Transactions = VariableList< pub type Withdrawals = VariableList::MaxWithdrawalsPerPayload>; #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Deneb), variant_attributes( derive( Default, @@ -77,16 +77,16 @@ pub struct ExecutionPayload { #[serde(with = "eth2_serde_utils::quoted_u256")] #[superstruct(getter(copy))] pub base_fee_per_gas: Uint256, - #[superstruct(only(Eip4844))] - #[serde(with = "eth2_serde_utils::quoted_u256")] - #[superstruct(getter(copy))] - pub excess_data_gas: Uint256, #[superstruct(getter(copy))] pub block_hash: ExecutionBlockHash, #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Deneb))] pub withdrawals: Withdrawals, + #[superstruct(only(Deneb))] + #[serde(with = "eth2_serde_utils::quoted_u256")] + #[superstruct(getter(copy))] + pub excess_data_gas: Uint256, } impl<'a, T: EthSpec> ExecutionPayloadRef<'a, T> { @@ -107,7 +107,7 @@ impl ExecutionPayload { ))), ForkName::Merge => ExecutionPayloadMerge::from_ssz_bytes(bytes).map(Self::Merge), ForkName::Capella => ExecutionPayloadCapella::from_ssz_bytes(bytes).map(Self::Capella), - ForkName::Eip4844 => ExecutionPayloadEip4844::from_ssz_bytes(bytes).map(Self::Eip4844), + ForkName::Deneb => ExecutionPayloadDeneb::from_ssz_bytes(bytes).map(Self::Deneb), } } @@ -137,9 +137,9 @@ impl ExecutionPayload { #[allow(clippy::integer_arithmetic)] /// Returns the maximum size of an execution payload. - pub fn max_execution_payload_eip4844_size() -> usize { + pub fn max_execution_payload_deneb_size() -> usize { // Fixed part - ExecutionPayloadEip4844::::default().as_ssz_bytes().len() + ExecutionPayloadDeneb::::default().as_ssz_bytes().len() // Max size of variable length `extra_data` field + (T::max_extra_data_bytes() * ::ssz_fixed_len()) // Max size of variable length `transactions` field @@ -161,7 +161,7 @@ impl ForkVersionDeserialize for ExecutionPayload { Ok(match fork_name { ForkName::Merge => Self::Merge(serde_json::from_value(value).map_err(convert_err)?), ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Eip4844 => Self::Eip4844(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), ForkName::Base | ForkName::Altair => { return Err(serde::de::Error::custom(format!( "ExecutionPayload failed to deserialize: unsupported fork '{}'", @@ -177,7 +177,7 @@ impl ExecutionPayload { match self { ExecutionPayload::Merge(_) => ForkName::Merge, ExecutionPayload::Capella(_) => ForkName::Capella, - ExecutionPayload::Eip4844(_) => ForkName::Eip4844, + ExecutionPayload::Deneb(_) => ForkName::Deneb, } } } diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 4dc79ddc999..bc1acc0ba97 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -9,7 +9,7 @@ use tree_hash_derive::TreeHash; use BeaconStateError; #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Deneb), variant_attributes( derive( Default, @@ -70,17 +70,17 @@ pub struct ExecutionPayloadHeader { #[serde(with = "eth2_serde_utils::quoted_u256")] #[superstruct(getter(copy))] pub base_fee_per_gas: Uint256, - #[superstruct(only(Eip4844))] - #[serde(with = "eth2_serde_utils::quoted_u256")] - #[superstruct(getter(copy))] - pub excess_data_gas: Uint256, #[superstruct(getter(copy))] pub block_hash: ExecutionBlockHash, #[superstruct(getter(copy))] pub transactions_root: Hash256, - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Deneb))] #[superstruct(getter(copy))] pub withdrawals_root: Hash256, + #[superstruct(only(Deneb))] + #[serde(with = "eth2_serde_utils::quoted_u256")] + #[superstruct(getter(copy))] + pub excess_data_gas: Uint256, } impl ExecutionPayloadHeader { @@ -97,9 +97,7 @@ impl ExecutionPayloadHeader { ForkName::Capella => { ExecutionPayloadHeaderCapella::from_ssz_bytes(bytes).map(Self::Capella) } - ForkName::Eip4844 => { - ExecutionPayloadHeaderEip4844::from_ssz_bytes(bytes).map(Self::Eip4844) - } + ForkName::Deneb => ExecutionPayloadHeaderDeneb::from_ssz_bytes(bytes).map(Self::Deneb), } } } @@ -136,8 +134,8 @@ impl ExecutionPayloadHeaderMerge { } impl ExecutionPayloadHeaderCapella { - pub fn upgrade_to_eip4844(&self) -> ExecutionPayloadHeaderEip4844 { - ExecutionPayloadHeaderEip4844 { + pub fn upgrade_to_deneb(&self) -> ExecutionPayloadHeaderDeneb { + ExecutionPayloadHeaderDeneb { parent_hash: self.parent_hash, fee_recipient: self.fee_recipient, state_root: self.state_root, @@ -150,11 +148,11 @@ impl ExecutionPayloadHeaderCapella { timestamp: self.timestamp, extra_data: self.extra_data.clone(), base_fee_per_gas: self.base_fee_per_gas, - // TODO: verify if this is correct - excess_data_gas: Uint256::zero(), block_hash: self.block_hash, transactions_root: self.transactions_root, withdrawals_root: self.withdrawals_root, + // TODO: verify if this is correct + excess_data_gas: Uint256::zero(), } } } @@ -201,8 +199,8 @@ impl<'a, T: EthSpec> From<&'a ExecutionPayloadCapella> for ExecutionPayloadHe } } -impl<'a, T: EthSpec> From<&'a ExecutionPayloadEip4844> for ExecutionPayloadHeaderEip4844 { - fn from(payload: &'a ExecutionPayloadEip4844) -> Self { +impl<'a, T: EthSpec> From<&'a ExecutionPayloadDeneb> for ExecutionPayloadHeaderDeneb { + fn from(payload: &'a ExecutionPayloadDeneb) -> Self { Self { parent_hash: payload.parent_hash, fee_recipient: payload.fee_recipient, @@ -216,10 +214,10 @@ impl<'a, T: EthSpec> From<&'a ExecutionPayloadEip4844> for ExecutionPayloadHe timestamp: payload.timestamp, extra_data: payload.extra_data.clone(), base_fee_per_gas: payload.base_fee_per_gas, - excess_data_gas: payload.excess_data_gas, block_hash: payload.block_hash, transactions_root: payload.transactions.tree_hash_root(), withdrawals_root: payload.withdrawals.tree_hash_root(), + excess_data_gas: payload.excess_data_gas, } } } @@ -238,7 +236,7 @@ impl<'a, T: EthSpec> From<&'a Self> for ExecutionPayloadHeaderCapella { } } -impl<'a, T: EthSpec> From<&'a Self> for ExecutionPayloadHeaderEip4844 { +impl<'a, T: EthSpec> From<&'a Self> for ExecutionPayloadHeaderDeneb { fn from(payload: &'a Self) -> Self { payload.clone() } @@ -274,13 +272,11 @@ impl TryFrom> for ExecutionPayloadHeaderCa } } } -impl TryFrom> for ExecutionPayloadHeaderEip4844 { +impl TryFrom> for ExecutionPayloadHeaderDeneb { type Error = BeaconStateError; fn try_from(header: ExecutionPayloadHeader) -> Result { match header { - ExecutionPayloadHeader::Eip4844(execution_payload_header) => { - Ok(execution_payload_header) - } + ExecutionPayloadHeader::Deneb(execution_payload_header) => Ok(execution_payload_header), _ => Err(BeaconStateError::IncorrectStateVariant), } } @@ -301,7 +297,7 @@ impl ForkVersionDeserialize for ExecutionPayloadHeader { Ok(match fork_name { ForkName::Merge => Self::Merge(serde_json::from_value(value).map_err(convert_err)?), ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), - ForkName::Eip4844 => Self::Eip4844(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), ForkName::Base | ForkName::Altair => { return Err(serde::de::Error::custom(format!( "ExecutionPayloadHeader failed to deserialize: unsupported fork '{}'", diff --git a/consensus/types/src/fork_context.rs b/consensus/types/src/fork_context.rs index f5221dd913d..23163f0eecb 100644 --- a/consensus/types/src/fork_context.rs +++ b/consensus/types/src/fork_context.rs @@ -54,10 +54,10 @@ impl ForkContext { )); } - if spec.eip4844_fork_epoch.is_some() { + if spec.deneb_fork_epoch.is_some() { fork_to_digest.push(( - ForkName::Eip4844, - ChainSpec::compute_fork_digest(spec.eip4844_fork_version, genesis_validators_root), + ForkName::Deneb, + ChainSpec::compute_fork_digest(spec.deneb_fork_version, genesis_validators_root), )); } diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index 89eaff7985d..e7c1f9628bc 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -12,7 +12,7 @@ pub enum ForkName { Altair, Merge, Capella, - Eip4844, + Deneb, } impl ForkName { @@ -22,7 +22,7 @@ impl ForkName { ForkName::Altair, ForkName::Merge, ForkName::Capella, - ForkName::Eip4844, + ForkName::Deneb, ] } @@ -35,35 +35,35 @@ impl ForkName { spec.altair_fork_epoch = None; spec.bellatrix_fork_epoch = None; spec.capella_fork_epoch = None; - spec.eip4844_fork_epoch = None; + spec.deneb_fork_epoch = None; spec } ForkName::Altair => { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = None; spec.capella_fork_epoch = None; - spec.eip4844_fork_epoch = None; + spec.deneb_fork_epoch = None; spec } ForkName::Merge => { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec.capella_fork_epoch = None; - spec.eip4844_fork_epoch = None; + spec.deneb_fork_epoch = None; spec } ForkName::Capella => { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec.capella_fork_epoch = Some(Epoch::new(0)); - spec.eip4844_fork_epoch = None; + spec.deneb_fork_epoch = None; spec } - ForkName::Eip4844 => { + ForkName::Deneb => { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec.capella_fork_epoch = Some(Epoch::new(0)); - spec.eip4844_fork_epoch = Some(Epoch::new(0)); + spec.deneb_fork_epoch = Some(Epoch::new(0)); spec } } @@ -78,7 +78,7 @@ impl ForkName { ForkName::Altair => Some(ForkName::Base), ForkName::Merge => Some(ForkName::Altair), ForkName::Capella => Some(ForkName::Merge), - ForkName::Eip4844 => Some(ForkName::Capella), + ForkName::Deneb => Some(ForkName::Capella), } } @@ -90,8 +90,8 @@ impl ForkName { ForkName::Base => Some(ForkName::Altair), ForkName::Altair => Some(ForkName::Merge), ForkName::Merge => Some(ForkName::Capella), - ForkName::Capella => Some(ForkName::Eip4844), - ForkName::Eip4844 => None, + ForkName::Capella => Some(ForkName::Deneb), + ForkName::Deneb => None, } } } @@ -137,9 +137,9 @@ macro_rules! map_fork_name_with { let (value, extra_data) = $body; ($t::Capella(value), extra_data) } - ForkName::Eip4844 => { + ForkName::Deneb => { let (value, extra_data) = $body; - ($t::Eip4844(value), extra_data) + ($t::Deneb(value), extra_data) } } }; @@ -154,7 +154,7 @@ impl FromStr for ForkName { "altair" => ForkName::Altair, "bellatrix" | "merge" => ForkName::Merge, "capella" => ForkName::Capella, - "eip4844" => ForkName::Eip4844, + "deneb" => ForkName::Deneb, _ => return Err(format!("unknown fork name: {}", fork_name)), }) } @@ -167,7 +167,7 @@ impl Display for ForkName { ForkName::Altair => "altair".fmt(f), ForkName::Merge => "bellatrix".fmt(f), ForkName::Capella => "capella".fmt(f), - ForkName::Eip4844 => "eip4844".fmt(f), + ForkName::Deneb => "deneb".fmt(f), } } } @@ -199,7 +199,7 @@ mod test { #[test] fn previous_and_next_fork_consistent() { - assert_eq!(ForkName::Eip4844.next_fork(), None); + assert_eq!(ForkName::Deneb.next_fork(), None); assert_eq!(ForkName::Base.previous_fork(), None); for (prev_fork, fork) in ForkName::list_all().into_iter().tuple_windows() { diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 14f3ff3560b..36a4e7e331c 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -109,12 +109,12 @@ pub use crate::attestation_data::AttestationData; pub use crate::attestation_duty::AttestationDuty; pub use crate::attester_slashing::AttesterSlashing; pub use crate::beacon_block::{ - BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockEip4844, + BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockDeneb, BeaconBlockMerge, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, EmptyBlock, }; pub use crate::beacon_block_body::{ BeaconBlockBody, BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyCapella, - BeaconBlockBodyEip4844, BeaconBlockBodyMerge, BeaconBlockBodyRef, BeaconBlockBodyRefMut, + BeaconBlockBodyDeneb, BeaconBlockBodyMerge, BeaconBlockBodyRef, BeaconBlockBodyRefMut, }; pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; @@ -137,11 +137,11 @@ pub use crate::eth_spec::EthSpecId; pub use crate::execution_block_hash::ExecutionBlockHash; pub use crate::execution_block_header::ExecutionBlockHeader; pub use crate::execution_payload::{ - ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, + ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, ExecutionPayloadRef, Transaction, Transactions, Withdrawals, }; pub use crate::execution_payload_header::{ - ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderEip4844, + ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRef, ExecutionPayloadHeaderRefMut, }; pub use crate::fork::Fork; @@ -159,9 +159,9 @@ pub use crate::light_client_optimistic_update::LightClientOptimisticUpdate; pub use crate::participation_flags::ParticipationFlags; pub use crate::participation_list::ParticipationList; pub use crate::payload::{ - AbstractExecPayload, BlindedPayload, BlindedPayloadCapella, BlindedPayloadEip4844, + AbstractExecPayload, BlindedPayload, BlindedPayloadCapella, BlindedPayloadDeneb, BlindedPayloadMerge, BlindedPayloadRef, BlockType, ExecPayload, FullPayload, - FullPayloadCapella, FullPayloadEip4844, FullPayloadMerge, FullPayloadRef, OwnedExecPayload, + FullPayloadCapella, FullPayloadDeneb, FullPayloadMerge, FullPayloadRef, OwnedExecPayload, }; pub use crate::pending_attestation::PendingAttestation; pub use crate::preset::{AltairPreset, BasePreset, BellatrixPreset, CapellaPreset}; @@ -173,7 +173,7 @@ pub use crate::shuffling_id::AttestationShufflingId; pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof; pub use crate::signed_beacon_block::{ SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, - SignedBeaconBlockEip4844, SignedBeaconBlockHash, SignedBeaconBlockMerge, + SignedBeaconBlockDeneb, SignedBeaconBlockHash, SignedBeaconBlockMerge, SignedBlindedBeaconBlock, }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 9ec2d9f30a2..fd15bb5d5dc 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -81,13 +81,13 @@ pub trait AbstractExecPayload: + TryFrom> + TryInto + TryInto - + TryInto + + TryInto { type Ref<'a>: ExecPayload + Copy + From<&'a Self::Merge> + From<&'a Self::Capella> - + From<&'a Self::Eip4844>; + + From<&'a Self::Deneb>; type Merge: OwnedExecPayload + Into @@ -97,16 +97,16 @@ pub trait AbstractExecPayload: + Into + for<'a> From>> + TryFrom>; - type Eip4844: OwnedExecPayload + type Deneb: OwnedExecPayload + Into - + for<'a> From>> - + TryFrom>; + + for<'a> From>> + + TryFrom>; fn default_at_fork(fork_name: ForkName) -> Result; } #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Deneb), variant_attributes( derive( Debug, @@ -145,8 +145,8 @@ pub struct FullPayload { pub execution_payload: ExecutionPayloadMerge, #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] pub execution_payload: ExecutionPayloadCapella, - #[superstruct(only(Eip4844), partial_getter(rename = "execution_payload_eip4844"))] - pub execution_payload: ExecutionPayloadEip4844, + #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] + pub execution_payload: ExecutionPayloadDeneb, } impl From> for ExecutionPayload { @@ -250,7 +250,7 @@ impl ExecPayload for FullPayload { FullPayload::Capella(ref inner) => { Ok(inner.execution_payload.withdrawals.tree_hash_root()) } - FullPayload::Eip4844(ref inner) => { + FullPayload::Deneb(ref inner) => { Ok(inner.execution_payload.withdrawals.tree_hash_root()) } } @@ -359,7 +359,7 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { FullPayloadRef::Capella(inner) => { Ok(inner.execution_payload.withdrawals.tree_hash_root()) } - FullPayloadRef::Eip4844(inner) => { + FullPayloadRef::Deneb(inner) => { Ok(inner.execution_payload.withdrawals.tree_hash_root()) } } @@ -382,14 +382,14 @@ impl AbstractExecPayload for FullPayload { type Ref<'a> = FullPayloadRef<'a, T>; type Merge = FullPayloadMerge; type Capella = FullPayloadCapella; - type Eip4844 = FullPayloadEip4844; + type Deneb = FullPayloadDeneb; fn default_at_fork(fork_name: ForkName) -> Result { match fork_name { ForkName::Base | ForkName::Altair => Err(Error::IncorrectStateVariant), ForkName::Merge => Ok(FullPayloadMerge::default().into()), ForkName::Capella => Ok(FullPayloadCapella::default().into()), - ForkName::Eip4844 => Ok(FullPayloadEip4844::default().into()), + ForkName::Deneb => Ok(FullPayloadDeneb::default().into()), } } } @@ -410,7 +410,7 @@ impl TryFrom> for FullPayload { } #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Deneb), variant_attributes( derive( Debug, @@ -448,8 +448,8 @@ pub struct BlindedPayload { pub execution_payload_header: ExecutionPayloadHeaderMerge, #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] pub execution_payload_header: ExecutionPayloadHeaderCapella, - #[superstruct(only(Eip4844), partial_getter(rename = "execution_payload_eip4844"))] - pub execution_payload_header: ExecutionPayloadHeaderEip4844, + #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] + pub execution_payload_header: ExecutionPayloadHeaderDeneb, } impl<'a, T: EthSpec> From> for BlindedPayload { @@ -531,9 +531,7 @@ impl ExecPayload for BlindedPayload { BlindedPayload::Capella(ref inner) => { Ok(inner.execution_payload_header.withdrawals_root) } - BlindedPayload::Eip4844(ref inner) => { - Ok(inner.execution_payload_header.withdrawals_root) - } + BlindedPayload::Deneb(ref inner) => Ok(inner.execution_payload_header.withdrawals_root), } } @@ -621,9 +619,7 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { BlindedPayloadRef::Capella(inner) => { Ok(inner.execution_payload_header.withdrawals_root) } - BlindedPayloadRef::Eip4844(inner) => { - Ok(inner.execution_payload_header.withdrawals_root) - } + BlindedPayloadRef::Deneb(inner) => Ok(inner.execution_payload_header.withdrawals_root), } } @@ -888,25 +884,25 @@ impl_exec_payload_for_fork!( Capella ); impl_exec_payload_for_fork!( - BlindedPayloadEip4844, - FullPayloadEip4844, - ExecutionPayloadHeaderEip4844, - ExecutionPayloadEip4844, - Eip4844 + BlindedPayloadDeneb, + FullPayloadDeneb, + ExecutionPayloadHeaderDeneb, + ExecutionPayloadDeneb, + Deneb ); impl AbstractExecPayload for BlindedPayload { type Ref<'a> = BlindedPayloadRef<'a, T>; type Merge = BlindedPayloadMerge; type Capella = BlindedPayloadCapella; - type Eip4844 = BlindedPayloadEip4844; + type Deneb = BlindedPayloadDeneb; fn default_at_fork(fork_name: ForkName) -> Result { match fork_name { ForkName::Base | ForkName::Altair => Err(Error::IncorrectStateVariant), ForkName::Merge => Ok(BlindedPayloadMerge::default().into()), ForkName::Capella => Ok(BlindedPayloadCapella::default().into()), - ForkName::Eip4844 => Ok(BlindedPayloadEip4844::default().into()), + ForkName::Deneb => Ok(BlindedPayloadDeneb::default().into()), } } } @@ -935,8 +931,8 @@ impl From> for BlindedPayload { execution_payload_header, }) } - ExecutionPayloadHeader::Eip4844(execution_payload_header) => { - Self::Eip4844(BlindedPayloadEip4844 { + ExecutionPayloadHeader::Deneb(execution_payload_header) => { + Self::Deneb(BlindedPayloadDeneb { execution_payload_header, }) } @@ -953,8 +949,8 @@ impl From> for ExecutionPayloadHeader { BlindedPayload::Capella(blinded_payload) => { ExecutionPayloadHeader::Capella(blinded_payload.execution_payload_header) } - BlindedPayload::Eip4844(blinded_payload) => { - ExecutionPayloadHeader::Eip4844(blinded_payload.execution_payload_header) + BlindedPayload::Deneb(blinded_payload) => { + ExecutionPayloadHeader::Deneb(blinded_payload.execution_payload_header) } } } diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index 301cfd5f878..58810150c21 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -37,7 +37,7 @@ impl From for Hash256 { /// A `BeaconBlock` and a signature from its proposer. #[superstruct( - variants(Base, Altair, Merge, Capella, Eip4844), + variants(Base, Altair, Merge, Capella, Deneb), variant_attributes( derive( Debug, @@ -76,8 +76,8 @@ pub struct SignedBeaconBlock = FullP pub message: BeaconBlockMerge, #[superstruct(only(Capella), partial_getter(rename = "message_capella"))] pub message: BeaconBlockCapella, - #[superstruct(only(Eip4844), partial_getter(rename = "message_eip4844"))] - pub message: BeaconBlockEip4844, + #[superstruct(only(Deneb), partial_getter(rename = "message_deneb"))] + pub message: BeaconBlockDeneb, pub signature: Signature, } @@ -138,8 +138,8 @@ impl> SignedBeaconBlock BeaconBlock::Capella(message) => { SignedBeaconBlock::Capella(SignedBeaconBlockCapella { message, signature }) } - BeaconBlock::Eip4844(message) => { - SignedBeaconBlock::Eip4844(SignedBeaconBlockEip4844 { message, signature }) + BeaconBlock::Deneb(message) => { + SignedBeaconBlock::Deneb(SignedBeaconBlockDeneb { message, signature }) } } } @@ -378,20 +378,20 @@ impl SignedBeaconBlockCapella> { } } -impl SignedBeaconBlockEip4844> { +impl SignedBeaconBlockDeneb> { pub fn into_full_block( self, - execution_payload: ExecutionPayloadEip4844, - ) -> SignedBeaconBlockEip4844> { - let SignedBeaconBlockEip4844 { + execution_payload: ExecutionPayloadDeneb, + ) -> SignedBeaconBlockDeneb> { + let SignedBeaconBlockDeneb { message: - BeaconBlockEip4844 { + BeaconBlockDeneb { slot, proposer_index, parent_root, state_root, body: - BeaconBlockBodyEip4844 { + BeaconBlockBodyDeneb { randao_reveal, eth1_data, graffiti, @@ -401,20 +401,20 @@ impl SignedBeaconBlockEip4844> { deposits, voluntary_exits, sync_aggregate, - execution_payload: BlindedPayloadEip4844 { .. }, + execution_payload: BlindedPayloadDeneb { .. }, bls_to_execution_changes, blob_kzg_commitments, }, }, signature, } = self; - SignedBeaconBlockEip4844 { - message: BeaconBlockEip4844 { + SignedBeaconBlockDeneb { + message: BeaconBlockDeneb { slot, proposer_index, parent_root, state_root, - body: BeaconBlockBodyEip4844 { + body: BeaconBlockBodyDeneb { randao_reveal, eth1_data, graffiti, @@ -424,7 +424,7 @@ impl SignedBeaconBlockEip4844> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayloadEip4844 { execution_payload }, + execution_payload: FullPayloadDeneb { execution_payload }, bls_to_execution_changes, blob_kzg_commitments, }, @@ -448,14 +448,14 @@ impl SignedBeaconBlock> { (SignedBeaconBlock::Capella(block), Some(ExecutionPayload::Capella(payload))) => { SignedBeaconBlock::Capella(block.into_full_block(payload)) } - (SignedBeaconBlock::Eip4844(block), Some(ExecutionPayload::Eip4844(payload))) => { - SignedBeaconBlock::Eip4844(block.into_full_block(payload)) + (SignedBeaconBlock::Deneb(block), Some(ExecutionPayload::Deneb(payload))) => { + SignedBeaconBlock::Deneb(block.into_full_block(payload)) } // avoid wildcard matching forks so that compiler will // direct us here when a new fork has been added (SignedBeaconBlock::Merge(_), _) => return None, (SignedBeaconBlock::Capella(_), _) => return None, - (SignedBeaconBlock::Eip4844(_), _) => return None, + (SignedBeaconBlock::Deneb(_), _) => return None, }; Some(full_block) } diff --git a/lcli/src/create_payload_header.rs b/lcli/src/create_payload_header.rs index 7700f23d9dd..5c96035851e 100644 --- a/lcli/src/create_payload_header.rs +++ b/lcli/src/create_payload_header.rs @@ -5,7 +5,7 @@ use std::fs::File; use std::io::Write; use std::time::{SystemTime, UNIX_EPOCH}; use types::{ - EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderEip4844, + EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderMerge, ForkName, }; @@ -40,13 +40,13 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { prev_randao: eth1_block_hash.into_root(), ..ExecutionPayloadHeaderCapella::default() }), - ForkName::Eip4844 => ExecutionPayloadHeader::Eip4844(ExecutionPayloadHeaderEip4844 { + ForkName::Deneb => ExecutionPayloadHeader::Deneb(ExecutionPayloadHeaderDeneb { gas_limit, base_fee_per_gas, timestamp: genesis_time, block_hash: eth1_block_hash, prev_randao: eth1_block_hash.into_root(), - ..ExecutionPayloadHeaderEip4844::default() + ..ExecutionPayloadHeaderDeneb::default() }), }; diff --git a/lcli/src/main.rs b/lcli/src/main.rs index 92ecc78ed16..1b509f17819 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -425,7 +425,7 @@ fn main() { .takes_value(true) .default_value("bellatrix") .help("The fork for which the execution payload header should be created.") - .possible_values(&["merge", "bellatrix", "capella", "eip4844"]) + .possible_values(&["merge", "bellatrix", "capella", "deneb"]) ) ) .subcommand( @@ -586,12 +586,12 @@ fn main() { ), ) .arg( - Arg::with_name("eip4844-fork-epoch") - .long("eip4844-fork-epoch") + Arg::with_name("deneb-fork-epoch") + .long("deneb-fork-epoch") .value_name("EPOCH") .takes_value(true) .help( - "The epoch at which to enable the eip4844 hard fork", + "The epoch at which to enable the deneb hard fork", ), ) .arg( diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index e28197e6178..8aef32f1a00 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -16,7 +16,7 @@ use types::ExecutionBlockHash; use types::{ test_utils::generate_deterministic_keypairs, Address, BeaconState, ChainSpec, Config, Epoch, Eth1Data, EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, - ExecutionPayloadHeaderEip4844, ExecutionPayloadHeaderMerge, ForkName, Hash256, Keypair, + ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderMerge, ForkName, Hash256, Keypair, PublicKey, Validator, }; @@ -82,8 +82,8 @@ pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul spec.capella_fork_epoch = Some(fork_epoch); } - if let Some(fork_epoch) = parse_optional(matches, "eip4844-fork-epoch")? { - spec.eip4844_fork_epoch = Some(fork_epoch); + if let Some(fork_epoch) = parse_optional(matches, "deneb-fork-epoch")? { + spec.deneb_fork_epoch = Some(fork_epoch); } if let Some(ttd) = parse_optional(matches, "ttd")? { @@ -112,9 +112,9 @@ pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul ExecutionPayloadHeaderCapella::::from_ssz_bytes(bytes.as_slice()) .map(ExecutionPayloadHeader::Capella) } - ForkName::Eip4844 => { - ExecutionPayloadHeaderEip4844::::from_ssz_bytes(bytes.as_slice()) - .map(ExecutionPayloadHeader::Eip4844) + ForkName::Deneb => { + ExecutionPayloadHeaderDeneb::::from_ssz_bytes(bytes.as_slice()) + .map(ExecutionPayloadHeader::Deneb) } } .map_err(|e| format!("SSZ decode failed: {:?}", e)) @@ -159,8 +159,8 @@ pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul None }; - let kzg_trusted_setup = if let Some(epoch) = spec.eip4844_fork_epoch { - // Only load the trusted setup if the eip4844 fork epoch is set + let kzg_trusted_setup = if let Some(epoch) = spec.deneb_fork_epoch { + // Only load the trusted setup if the deneb fork epoch is set if epoch != Epoch::max_value() { let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP) .map_err(|e| format!("Unable to read trusted setup file: {}", e))?; diff --git a/lcli/src/parse_ssz.rs b/lcli/src/parse_ssz.rs index 70f34e09e3c..d51ea3a2f15 100644 --- a/lcli/src/parse_ssz.rs +++ b/lcli/src/parse_ssz.rs @@ -52,17 +52,17 @@ pub fn run_parse_ssz(matches: &ArgMatches) -> Result<(), String> { "signed_block_altair" => decode_and_print::>(&bytes, format)?, "signed_block_merge" => decode_and_print::>(&bytes, format)?, "signed_block_capella" => decode_and_print::>(&bytes, format)?, - "signed_block_eip4844" => decode_and_print::>(&bytes, format)?, + "signed_block_deneb" => decode_and_print::>(&bytes, format)?, "block_base" => decode_and_print::>(&bytes, format)?, "block_altair" => decode_and_print::>(&bytes, format)?, "block_merge" => decode_and_print::>(&bytes, format)?, "block_capella" => decode_and_print::>(&bytes, format)?, - "block_eip4844" => decode_and_print::>(&bytes, format)?, + "block_deneb" => decode_and_print::>(&bytes, format)?, "state_base" => decode_and_print::>(&bytes, format)?, "state_altair" => decode_and_print::>(&bytes, format)?, "state_merge" => decode_and_print::>(&bytes, format)?, "state_capella" => decode_and_print::>(&bytes, format)?, - "state_eip4844" => decode_and_print::>(&bytes, format)?, + "state_deneb" => decode_and_print::>(&bytes, format)?, "blob_sidecar" => decode_and_print::>(&bytes, format)?, other => return Err(format!("Unknown type: {}", other)), }; diff --git a/scripts/local_testnet/setup.sh b/scripts/local_testnet/setup.sh index 1dc7566f6b2..5195d26750f 100755 --- a/scripts/local_testnet/setup.sh +++ b/scripts/local_testnet/setup.sh @@ -29,7 +29,7 @@ lcli \ --altair-fork-epoch $ALTAIR_FORK_EPOCH \ --bellatrix-fork-epoch $BELLATRIX_FORK_EPOCH \ --capella-fork-epoch $CAPELLA_FORK_EPOCH \ - --eip4844-fork-epoch $EIP4844_FORK_EPOCH \ + --deneb-fork-epoch $DENEB_FORK_EPOCH \ --ttd $TTD \ --eth1-block-hash $ETH1_BLOCK_HASH \ --eth1-id $CHAIN_ID \ @@ -54,7 +54,7 @@ echo Validators generated with keystore passwords at $DATADIR. GENESIS_TIME=$(lcli pretty-ssz state_merge ~/.lighthouse/local-testnet/testnet/genesis.ssz | jq | grep -Po 'genesis_time": "\K.*\d') CAPELLA_TIME=$((GENESIS_TIME + (CAPELLA_FORK_EPOCH * 32 * SECONDS_PER_SLOT))) -EIP4844_TIME=$((GENESIS_TIME + (EIP4844_FORK_EPOCH * 32 * SECONDS_PER_SLOT))) +DENEB_TIME=$((GENESIS_TIME + (DENEB_FORK_EPOCH * 32 * SECONDS_PER_SLOT))) sed -i 's/"shanghaiTime".*$/"shanghaiTime": '"$CAPELLA_TIME"',/g' genesis.json -sed -i 's/"shardingForkTime".*$/"shardingForkTime": '"$EIP4844_TIME"',/g' genesis.json +sed -i 's/"shardingForkTime".*$/"shardingForkTime": '"$DENEB_TIME"',/g' genesis.json diff --git a/scripts/local_testnet/vars.env b/scripts/local_testnet/vars.env index dcdf671e3f1..515153c785f 100644 --- a/scripts/local_testnet/vars.env +++ b/scripts/local_testnet/vars.env @@ -41,7 +41,7 @@ CHAIN_ID=4242 ALTAIR_FORK_EPOCH=0 BELLATRIX_FORK_EPOCH=0 CAPELLA_FORK_EPOCH=1 -EIP4844_FORK_EPOCH=2 +DENEB_FORK_EPOCH=2 TTD=0 diff --git a/scripts/tests/vars.env b/scripts/tests/vars.env index e7a688769a9..b656492b3cf 100644 --- a/scripts/tests/vars.env +++ b/scripts/tests/vars.env @@ -36,7 +36,7 @@ CHAIN_ID=4242 ALTAIR_FORK_EPOCH=18446744073709551615 BELLATRIX_FORK_EPOCH=18446744073709551615 CAPELLA_FORK_EPOCH=18446744073709551615 -EIP4844_FORK_EPOCH=18446744073709551615 +DENEB_FORK_EPOCH=18446744073709551615 # Spec version (mainnet or minimal) SPEC_PRESET=mainnet diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index eb2aa10307e..e663ead1ab5 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,4 +1,4 @@ -TESTS_TAG := v1.3.0-rc.1 # FIXME: move to latest +TESTS_TAG :=v1.3.0-rc.5 TESTS = general minimal mainnet TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index 10c22f0a965..e1944aaf45e 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -50,7 +50,11 @@ # some bls tests are not included now "bls12-381-tests/deserialization_G1", "bls12-381-tests/deserialization_G2", - "bls12-381-tests/hash_to_G2" + "bls12-381-tests/hash_to_G2", + # FIXME(sean) + "tests/mainnet/capella/light_client/single_merkle_proof/BeaconBlockBody/*", + "tests/mainnet/deneb/light_client/single_merkle_proof/BeaconBlockBody/*", + "tests/general/deneb/kzg" ] def normalize_path(path): diff --git a/testing/ef_tests/src/cases/common.rs b/testing/ef_tests/src/cases/common.rs index f889002bd88..68fd990f1fb 100644 --- a/testing/ef_tests/src/cases/common.rs +++ b/testing/ef_tests/src/cases/common.rs @@ -66,7 +66,7 @@ pub fn previous_fork(fork_name: ForkName) -> ForkName { ForkName::Altair => ForkName::Base, ForkName::Merge => ForkName::Altair, ForkName::Capella => ForkName::Merge, - ForkName::Eip4844 => ForkName::Capella, + ForkName::Deneb => ForkName::Capella, } } diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index 8b04319f648..34c4a74a9b6 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -104,7 +104,7 @@ impl EpochTransition for JustificationAndFinalization { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => { + | BeaconState::Deneb(_) => { let justification_and_finalization_state = altair::process_justification_and_finalization( state, @@ -128,7 +128,7 @@ impl EpochTransition for RewardsAndPenalties { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => altair::process_rewards_and_penalties( + | BeaconState::Deneb(_) => altair::process_rewards_and_penalties( state, &altair::ParticipationCache::new(state, spec).unwrap(), spec, @@ -158,7 +158,7 @@ impl EpochTransition for Slashings { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => { + | BeaconState::Deneb(_) => { process_slashings( state, altair::ParticipationCache::new(state, spec) @@ -210,7 +210,7 @@ impl EpochTransition for HistoricalRootsUpdate { impl EpochTransition for HistoricalSummariesUpdate { fn run(state: &mut BeaconState, _spec: &ChainSpec) -> Result<(), EpochProcessingError> { match state { - BeaconState::Capella(_) | BeaconState::Eip4844(_) => { + BeaconState::Capella(_) | BeaconState::Deneb(_) => { process_historical_summaries_update(state) } _ => Ok(()), @@ -235,7 +235,7 @@ impl EpochTransition for SyncCommitteeUpdates { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => altair::process_sync_committee_updates(state, spec), + | BeaconState::Deneb(_) => altair::process_sync_committee_updates(state, spec), } } } @@ -247,7 +247,7 @@ impl EpochTransition for InactivityUpdates { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => altair::process_inactivity_updates( + | BeaconState::Deneb(_) => altair::process_inactivity_updates( state, &altair::ParticipationCache::new(state, spec).unwrap(), spec, @@ -263,7 +263,7 @@ impl EpochTransition for ParticipationFlagUpdates { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => altair::process_participation_flag_updates(state), + | BeaconState::Deneb(_) => altair::process_participation_flag_updates(state), } } } @@ -314,7 +314,7 @@ impl> Case for EpochProcessing { T::name() != "participation_record_updates" && T::name() != "historical_summaries_update" } - ForkName::Capella | ForkName::Eip4844 => { + ForkName::Capella | ForkName::Deneb => { T::name() != "participation_record_updates" && T::name() != "historical_roots_update" } diff --git a/testing/ef_tests/src/cases/fork.rs b/testing/ef_tests/src/cases/fork.rs index 26939ce0447..d27fbcd6715 100644 --- a/testing/ef_tests/src/cases/fork.rs +++ b/testing/ef_tests/src/cases/fork.rs @@ -4,7 +4,7 @@ use crate::cases::common::previous_fork; use crate::decode::{ssz_decode_state, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::upgrade::{ - upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_eip4844, + upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, }; use types::{BeaconState, ForkName}; @@ -64,7 +64,7 @@ impl Case for ForkTest { ForkName::Altair => upgrade_to_altair(&mut result_state, spec).map(|_| result_state), ForkName::Merge => upgrade_to_bellatrix(&mut result_state, spec).map(|_| result_state), ForkName::Capella => upgrade_to_capella(&mut result_state, spec).map(|_| result_state), - ForkName::Eip4844 => upgrade_to_eip4844(&mut result_state, spec).map(|_| result_state), + ForkName::Deneb => upgrade_to_deneb(&mut result_state, spec).map(|_| result_state), }; compare_beacon_state_results_without_caches(&mut result, &mut expected) diff --git a/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs b/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/ef_tests/src/cases/merkle_proof_validity.rs b/testing/ef_tests/src/cases/merkle_proof_validity.rs index c180774bb64..cdcdaaf9b36 100644 --- a/testing/ef_tests/src/cases/merkle_proof_validity.rs +++ b/testing/ef_tests/src/cases/merkle_proof_validity.rs @@ -28,6 +28,11 @@ pub struct MerkleProofValidity { impl LoadCase for MerkleProofValidity { fn load_from_dir(path: &Path, fork_name: ForkName) -> Result { + //FIXME(sean) + if path.ends_with("execution_merkle_proof") { + return Err(Error::SkippedKnownFailure); + } + let spec = &testing_spec::(fork_name); let state = ssz_decode_state(&path.join("object.ssz_snappy"), spec)?; let merkle_proof = yaml_decode_file(&path.join("proof.yaml"))?; diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 71954405c0c..2f27b43a1df 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -98,7 +98,7 @@ impl Operation for Attestation { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => { + | BeaconState::Deneb(_) => { altair::process_attestation(state, self, 0, &mut ctxt, VerifySignatures::True, spec) } } diff --git a/testing/ef_tests/src/cases/transition.rs b/testing/ef_tests/src/cases/transition.rs index fb7ccfea644..4974eb881a1 100644 --- a/testing/ef_tests/src/cases/transition.rs +++ b/testing/ef_tests/src/cases/transition.rs @@ -47,11 +47,11 @@ impl LoadCase for TransitionTest { spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec.capella_fork_epoch = Some(metadata.fork_epoch); } - ForkName::Eip4844 => { + ForkName::Deneb => { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec.capella_fork_epoch = Some(Epoch::new(0)); - spec.eip4844_fork_epoch = Some(metadata.fork_epoch); + spec.deneb_fork_epoch = Some(metadata.fork_epoch); } } diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index 9a20c3a3e9c..23fb20cf201 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -1,6 +1,6 @@ use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation}; -use crate::type_name; use crate::type_name::TypeName; +use crate::{type_name, Error}; use derivative::Derivative; use std::fs::{self, DirEntry}; use std::marker::PhantomData; @@ -57,11 +57,17 @@ pub trait Handler { .filter_map(as_directory) .flat_map(|suite| fs::read_dir(suite.path()).expect("suite dir exists")) .filter_map(as_directory) - .map(|test_case_dir| { + .filter_map(|test_case_dir| { let path = test_case_dir.path(); - let case = Self::Case::load_from_dir(&path, fork_name).expect("test should load"); - (path, case) + let case_result = Self::Case::load_from_dir(&path, fork_name); + + if let Err(Error::SkippedKnownFailure) = case_result.as_ref() { + return None; + } + + let case = case_result.expect("test should load"); + Some((path, case)) }) .collect(); @@ -218,8 +224,8 @@ impl SszStaticHandler { Self::for_forks(vec![ForkName::Capella]) } - pub fn eip4844_only() -> Self { - Self::for_forks(vec![ForkName::Eip4844]) + pub fn deneb_only() -> Self { + Self::for_forks(vec![ForkName::Deneb]) } pub fn altair_and_later() -> Self { diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index d94dfef4854..ef128440301 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -1,4 +1,5 @@ //! Mapping from types to canonical string identifiers used in testing. +use types::blob_sidecar::BlobIdentifier; use types::historical_summary::HistoricalSummary; use types::*; @@ -47,9 +48,10 @@ type_name_generic!(BeaconBlockBodyBase, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyAltair, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyMerge, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyCapella, "BeaconBlockBody"); -type_name_generic!(BeaconBlockBodyEip4844, "BeaconBlockBody"); +type_name_generic!(BeaconBlockBodyDeneb, "BeaconBlockBody"); type_name!(BeaconBlockHeader); type_name_generic!(BeaconState); +type_name!(BlobIdentifier); type_name_generic!(BlobSidecar); type_name!(Checkpoint); type_name_generic!(ContributionAndProof); @@ -60,12 +62,12 @@ type_name!(Eth1Data); type_name_generic!(ExecutionPayload); type_name_generic!(ExecutionPayloadMerge, "ExecutionPayload"); type_name_generic!(ExecutionPayloadCapella, "ExecutionPayload"); -type_name_generic!(ExecutionPayloadEip4844, "ExecutionPayload"); +type_name_generic!(ExecutionPayloadDeneb, "ExecutionPayload"); type_name_generic!(FullPayload, "ExecutionPayload"); type_name_generic!(ExecutionPayloadHeader); type_name_generic!(ExecutionPayloadHeaderMerge, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderCapella, "ExecutionPayloadHeader"); -type_name_generic!(ExecutionPayloadHeaderEip4844, "ExecutionPayloadHeader"); +type_name_generic!(ExecutionPayloadHeaderDeneb, "ExecutionPayloadHeader"); type_name_generic!(BlindedPayload, "ExecutionPayloadHeader"); type_name!(Fork); type_name!(ForkData); @@ -76,6 +78,7 @@ type_name!(ProposerSlashing); type_name_generic!(SignedAggregateAndProof); type_name_generic!(SignedBeaconBlock); type_name!(SignedBeaconBlockHeader); +type_name_generic!(SignedBlobSidecar); type_name_generic!(SignedContributionAndProof); type_name!(SignedVoluntaryExit); type_name!(SigningData); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index d5b8dc80a61..2bd8580bf2a 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -215,6 +215,7 @@ macro_rules! ssz_static_test_no_run { #[cfg(feature = "fake_crypto")] mod ssz_static { use ef_tests::{Handler, SszStaticHandler, SszStaticTHCHandler, SszStaticWithSpecHandler}; + use types::blob_sidecar::BlobIdentifier; use types::historical_summary::HistoricalSummary; use types::*; @@ -267,9 +268,9 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::capella_only() .run(); - SszStaticHandler::, MinimalEthSpec>::eip4844_only() + SszStaticHandler::, MinimalEthSpec>::deneb_only() .run(); - SszStaticHandler::, MainnetEthSpec>::eip4844_only() + SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); } @@ -331,9 +332,9 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::capella_only() .run(); - SszStaticHandler::, MinimalEthSpec>::eip4844_only() + SszStaticHandler::, MinimalEthSpec>::deneb_only() .run(); - SszStaticHandler::, MainnetEthSpec>::eip4844_only() + SszStaticHandler::, MainnetEthSpec>::deneb_only() .run(); } @@ -347,10 +348,10 @@ mod ssz_static { ::capella_only().run(); SszStaticHandler::, MainnetEthSpec> ::capella_only().run(); - SszStaticHandler::, MinimalEthSpec> - ::eip4844_only().run(); - SszStaticHandler::, MainnetEthSpec> - ::eip4844_only().run(); + SszStaticHandler::, MinimalEthSpec> + ::deneb_only().run(); + SszStaticHandler::, MainnetEthSpec> + ::deneb_only().run(); } #[test] @@ -372,9 +373,21 @@ mod ssz_static { } #[test] - fn blobs_sidecar() { - SszStaticHandler::, MinimalEthSpec>::eip4844_only().run(); - SszStaticHandler::, MainnetEthSpec>::eip4844_only().run(); + fn blob_sidecar() { + SszStaticHandler::, MinimalEthSpec>::deneb_only().run(); + SszStaticHandler::, MainnetEthSpec>::deneb_only().run(); + } + + #[test] + fn signed_blob_sidecar() { + SszStaticHandler::, MinimalEthSpec>::deneb_only().run(); + SszStaticHandler::, MainnetEthSpec>::deneb_only().run(); + } + + #[test] + fn blob_identifier() { + SszStaticHandler::::deneb_only().run(); + SszStaticHandler::::deneb_only().run(); } #[test] diff --git a/validator_client/src/signing_method/web3signer.rs b/validator_client/src/signing_method/web3signer.rs index 512cbc7d023..3ea925144eb 100644 --- a/validator_client/src/signing_method/web3signer.rs +++ b/validator_client/src/signing_method/web3signer.rs @@ -27,7 +27,7 @@ pub enum ForkName { Altair, Bellatrix, Capella, - Eip4844, + Deneb, } #[derive(Debug, PartialEq, Serialize)] @@ -97,8 +97,8 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> Web3SignerObject<'a, T, Pa block: None, block_header: Some(block.block_header()), }), - BeaconBlock::Eip4844(_) => Ok(Web3SignerObject::BeaconBlock { - version: ForkName::Eip4844, + BeaconBlock::Deneb(_) => Ok(Web3SignerObject::BeaconBlock { + version: ForkName::Deneb, block: None, block_header: Some(block.block_header()), }), From af974dc0b89eb908caf9ba2391cd556f0e38f3ef Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 26 Mar 2023 19:18:54 -0400 Subject: [PATCH 32/96] use block wrapper in sync pairing (#4131) --- .../beacon_chain/src/blob_verification.rs | 2 +- .../src/data_availability_checker.rs | 4 +- beacon_node/http_api/src/publish_blocks.rs | 7 +-- .../src/sync/block_sidecar_coupling.rs | 8 +-- .../network/src/sync/network_context.rs | 55 ++++--------------- 5 files changed, 20 insertions(+), 56 deletions(-) diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 583d1a60b31..48ad45e83b3 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -446,7 +446,7 @@ impl AsBlock for &MaybeAvailableBlock { #[derivative(Hash(bound = "E: EthSpec"))] pub enum BlockWrapper { Block(Arc>), - BlockAndBlobs(Arc>, BlobSidecarList), + BlockAndBlobs(Arc>, Vec>>), } impl AsBlock for BlockWrapper { diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index b6c53e354e1..b2e2e609dce 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -240,7 +240,7 @@ impl DataAvailabilityChecker { .kzg .as_ref() .ok_or(AvailabilityCheckError::KzgNotInitialized)?; - let verified_blobs = verify_kzg_for_blob_list(blob_list, kzg)?; + let verified_blobs = verify_kzg_for_blob_list(VariableList::new(blob_list)?, kzg)?; Ok(MaybeAvailableBlock::Available( self.check_availability_with_blobs(block, verified_blobs)?, @@ -508,7 +508,7 @@ impl AsBlock for AvailableBlock { fn into_block_wrapper(self) -> BlockWrapper { let (block, blobs_opt) = self.deconstruct(); if let Some(blobs) = blobs_opt { - BlockWrapper::BlockAndBlobs(block, blobs) + BlockWrapper::BlockAndBlobs(block, blobs.to_vec()) } else { BlockWrapper::Block(block) } diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 4894663225e..d722cf6c9b7 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -4,7 +4,7 @@ use beacon_chain::blob_verification::{AsBlock, BlockWrapper}; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{AvailabilityProcessingStatus, NotifyExecutionLayer}; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockError, CountUnrealized}; -use eth2::types::{SignedBlockContents, VariableList}; +use eth2::types::SignedBlockContents; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; use network::NetworkMessage; @@ -77,10 +77,7 @@ pub async fn publish_block( PubsubMessage::BlobSidecar(Box::new((blob_index as u64, blob))), )?; } - let blobs_vec = signed_blobs.into_iter().map(|blob| blob.message).collect(); - let blobs = VariableList::new(blobs_vec).map_err(|e| { - warp_utils::reject::custom_server_error(format!("Invalid blobs length: {e:?}")) - })?; + let blobs = signed_blobs.into_iter().map(|blob| blob.message).collect(); BlockWrapper::BlockAndBlobs(block, blobs) } else { block.into() diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index 67db9a7a326..e6c5549cc9e 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -1,4 +1,4 @@ -use super::network_context::TempBlockWrapper; +use beacon_chain::blob_verification::BlockWrapper; use std::{collections::VecDeque, sync::Arc}; use types::{BlobSidecar, EthSpec, SignedBeaconBlock}; @@ -29,7 +29,7 @@ impl BlocksAndBlobsRequestInfo { } } - pub fn into_responses(self) -> Result>, &'static str> { + pub fn into_responses(self) -> Result>, &'static str> { let BlocksAndBlobsRequestInfo { accumulated_blocks, accumulated_sidecars, @@ -53,9 +53,9 @@ impl BlocksAndBlobsRequestInfo { } if blob_list.is_empty() { - responses.push(TempBlockWrapper::Block(block)) + responses.push(BlockWrapper::Block(block)) } else { - responses.push(TempBlockWrapper::BlockAndBlobList(block, blob_list)) + responses.push(BlockWrapper::BlockAndBlobs(block, blob_list)) } } diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 974d8dbd8c8..ced6aeb52ed 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -20,12 +20,6 @@ use std::sync::Arc; use tokio::sync::mpsc; use types::{BlobSidecar, EthSpec, SignedBeaconBlock}; -// Temporary struct to handle incremental changes in the meantime. -pub enum TempBlockWrapper { - Block(Arc>), - BlockAndBlobList(Arc>, Vec>>), -} - pub struct BlocksAndBlobsByRangeResponse { pub batch_id: BatchId, pub responses: Result>, &'static str>, @@ -328,26 +322,13 @@ impl SyncNetworkContext { batch_id, block_blob_info, } = entry.remove(); - - let responses = block_blob_info.into_responses(); - let unimplemented_info = match responses { - Ok(responses) => { - let infos = responses - .into_iter() - .map(|temp_block_wrapper| match temp_block_wrapper { - TempBlockWrapper::Block(block) => { - format!("slot{}", block.slot()) - } - TempBlockWrapper::BlockAndBlobList(block, blob_list) => { - format!("slot{}({} blobs)", block.slot(), blob_list.len()) - } - }) - .collect::>(); - infos.join(", ") - } - Err(e) => format!("Error: {e}"), - }; - unimplemented!("Here we are supposed to return a block possibly paired with a Bundle of blobs, but only have a list of individual blobs. This is what we got from the network: ChainId[{chain_id}] BatchId[{batch_id}] {unimplemented_info}") + Some(( + chain_id, + BlocksAndBlobsByRangeResponse { + batch_id, + responses: block_blob_info.into_responses(), + }, + )) } else { None } @@ -416,24 +397,10 @@ impl SyncNetworkContext { let (batch_id, info) = entry.remove(); let responses = info.into_responses(); - let unimplemented_info = match responses { - Ok(responses) => { - let infos = responses - .into_iter() - .map(|temp_block_wrapper| match temp_block_wrapper { - TempBlockWrapper::Block(block) => { - format!("slot{}", block.slot()) - } - TempBlockWrapper::BlockAndBlobList(block, blob_list) => { - format!("slot{}({} blobs)", block.slot(), blob_list.len()) - } - }) - .collect::>(); - infos.join(", ") - } - Err(e) => format!("Error: {e}"), - }; - unimplemented!("Here we are supposed to return a block possibly paired with a Bundle of blobs for backfill, but only have a list of individual blobs. This is what we got from the network: BatchId[{batch_id}]{unimplemented_info}") + Some(BlocksAndBlobsByRangeResponse { + batch_id, + responses, + }) } else { None } From f58086333710f48ecec5b5f26af64476fa7e3a7e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 27 Mar 2023 10:09:53 -0400 Subject: [PATCH 33/96] Add simple pruning to data availability checker (#4132) * use block wrapper in sync pairing * add pruning to the data availability checker * remove unused function, rename function --- .../beacon_chain/src/block_verification.rs | 42 +++++ .../src/data_availability_checker.rs | 143 ++++++++++-------- 2 files changed, 120 insertions(+), 65 deletions(-) diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 4cadc86bd23..7c137146865 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -93,6 +93,7 @@ use std::time::Duration; use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp}; use task_executor::JoinHandle; use tree_hash::TreeHash; +use types::blob_sidecar::BlobIdentifier; use types::ExecPayload; use types::{ BeaconBlockRef, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, CloneConfig, Epoch, @@ -735,6 +736,23 @@ impl AvailableExecutedBlock { payload_verification_outcome, } } + + pub fn get_all_blob_ids(&self) -> Vec { + let num_blobs_expected = self + .block + .message() + .body() + .blob_kzg_commitments() + .map_or(0, |commitments| commitments.len()); + let mut blob_ids = Vec::with_capacity(num_blobs_expected); + for i in 0..num_blobs_expected { + blob_ids.push(BlobIdentifier { + block_root: self.import_data.block_root, + index: i as u64, + }); + } + blob_ids + } } pub struct AvailabilityPendingExecutedBlock { @@ -755,6 +773,30 @@ impl AvailabilityPendingExecutedBlock { payload_verification_outcome, } } + + pub fn num_blobs_expected(&self) -> usize { + self.block + .kzg_commitments() + .map_or(0, |commitments| commitments.len()) + } + + pub fn get_all_blob_ids(&self) -> Vec { + self.get_filtered_blob_ids(|_| true) + } + + pub fn get_filtered_blob_ids(&self, filter: impl Fn(usize) -> bool) -> Vec { + let num_blobs_expected = self.num_blobs_expected(); + let mut blob_ids = Vec::with_capacity(num_blobs_expected); + for i in 0..num_blobs_expected { + if filter(i) { + blob_ids.push(BlobIdentifier { + block_root: self.import_data.block_root, + index: i as u64, + }); + } + } + blob_ids + } } pub struct BlockImportData { diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index b2e2e609dce..3046c8b396a 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -10,7 +10,7 @@ use parking_lot::{Mutex, RwLock}; use slot_clock::SlotClock; use ssz_types::{Error, VariableList}; use state_processing::per_block_processing::deneb::deneb::verify_kzg_commitments_against_transactions; -use std::collections::hash_map::Entry; +use std::collections::hash_map::{Entry, OccupiedEntry}; use std::collections::HashMap; use std::sync::Arc; use types::beacon_block_body::KzgCommitments; @@ -65,12 +65,42 @@ struct GossipBlobCache { executed_block: Option>, } +impl GossipBlobCache { + fn new_from_blob(blob: KzgVerifiedBlob) -> Self { + Self { + verified_blobs: vec![blob], + executed_block: None, + } + } + + fn new_from_block(block: AvailabilityPendingExecutedBlock) -> Self { + Self { + verified_blobs: vec![], + executed_block: Some(block), + } + } + + fn has_all_blobs(&self, block: &AvailabilityPendingExecutedBlock) -> bool { + self.verified_blobs.len() == block.num_blobs_expected() + } +} + pub enum Availability { PendingBlobs(Vec), PendingBlock(Hash256), Available(Box>), } +impl Availability { + pub fn get_available_blob_ids(&self) -> Option> { + if let Self::Available(block) = self { + Some(block.get_all_blob_ids()) + } else { + None + } + } +} + impl DataAvailabilityChecker { pub fn new(slot_clock: S, kzg: Option>, spec: ChainSpec) -> Self { Self { @@ -90,23 +120,19 @@ impl DataAvailabilityChecker { /// This should only accept gossip verified blobs, so we should not have to worry about dupes. pub fn put_gossip_blob( &self, - verified_blob: GossipVerifiedBlob, + gossip_blob: GossipVerifiedBlob, ) -> Result, AvailabilityCheckError> { - let block_root = verified_blob.block_root(); + let block_root = gossip_blob.block_root(); + // Verify the KZG commitments. let kzg_verified_blob = if let Some(kzg) = self.kzg.as_ref() { - verify_kzg_for_blob(verified_blob, kzg)? + verify_kzg_for_blob(gossip_blob, kzg)? } else { return Err(AvailabilityCheckError::KzgNotInitialized); }; - //TODO(sean) can we just use a referece to the blob here? let blob = kzg_verified_blob.clone_blob(); - // check if we have a block - // check if the complete set matches the block - // verify, otherwise cache - let mut blob_cache = self.gossip_blob_cache.lock(); // Gossip cache. @@ -121,25 +147,25 @@ impl DataAvailabilityChecker { .insert(blob.index as usize, kzg_verified_blob); if let Some(executed_block) = cache.executed_block.take() { - self.check_block_availability_or_cache(cache, executed_block)? + self.check_block_availability_maybe_cache(occupied_entry, executed_block)? } else { Availability::PendingBlock(block_root) } } Entry::Vacant(vacant_entry) => { let block_root = kzg_verified_blob.block_root(); - vacant_entry.insert(GossipBlobCache { - verified_blobs: vec![kzg_verified_blob], - executed_block: None, - }); + vacant_entry.insert(GossipBlobCache::new_from_blob(kzg_verified_blob)); Availability::PendingBlock(block_root) } }; drop(blob_cache); - // RPC cache. - self.rpc_blob_cache.write().insert(blob.id(), blob.clone()); + if let Some(blob_ids) = availability.get_available_blob_ids() { + self.prune_rpc_blob_cache(&blob_ids); + } else { + self.rpc_blob_cache.write().insert(blob.id(), blob.clone()); + } Ok(availability) } @@ -154,49 +180,40 @@ impl DataAvailabilityChecker { let entry = guard.entry(executed_block.import_data.block_root); let availability = match entry { - Entry::Occupied(mut occupied_entry) => { - let cache: &mut GossipBlobCache = occupied_entry.get_mut(); - - self.check_block_availability_or_cache(cache, executed_block)? + Entry::Occupied(occupied_entry) => { + self.check_block_availability_maybe_cache(occupied_entry, executed_block)? } Entry::Vacant(vacant_entry) => { - let kzg_commitments_len = executed_block.block.kzg_commitments()?.len(); - let mut blob_ids = Vec::with_capacity(kzg_commitments_len); - for i in 0..kzg_commitments_len { - blob_ids.push(BlobIdentifier { - block_root: executed_block.import_data.block_root, - index: i as u64, - }); - } - - vacant_entry.insert(GossipBlobCache { - verified_blobs: vec![], - executed_block: Some(executed_block), - }); - - Availability::PendingBlobs(blob_ids) + let all_blob_ids = executed_block.get_all_blob_ids(); + vacant_entry.insert(GossipBlobCache::new_from_block(executed_block)); + Availability::PendingBlobs(all_blob_ids) } }; + drop(guard); + + if let Some(blob_ids) = availability.get_available_blob_ids() { + self.prune_rpc_blob_cache(&blob_ids); + } + Ok(availability) } - fn check_block_availability_or_cache( + fn check_block_availability_maybe_cache( &self, - cache: &mut GossipBlobCache, + mut occupied_entry: OccupiedEntry>, executed_block: AvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { - let AvailabilityPendingExecutedBlock { - block, - import_data, - payload_verification_outcome, - } = executed_block; - let kzg_commitments_len = block.kzg_commitments()?.len(); - let verified_commitments_len = cache.verified_blobs.len(); - if kzg_commitments_len == verified_commitments_len { - //TODO(sean) can we remove this clone - let blobs = cache.verified_blobs.clone(); - let available_block = self.make_available(block, blobs)?; + if occupied_entry.get().has_all_blobs(&executed_block) { + let AvailabilityPendingExecutedBlock { + block, + import_data, + payload_verification_outcome, + } = executed_block; + + let cache = occupied_entry.remove(); + + let available_block = self.make_available(block, cache.verified_blobs)?; Ok(Availability::Available(Box::new( AvailableExecutedBlock::new( available_block, @@ -205,25 +222,14 @@ impl DataAvailabilityChecker { ), ))) } else { - let mut missing_blobs = Vec::with_capacity(kzg_commitments_len); - for i in 0..kzg_commitments_len { - if cache.verified_blobs.get(i).is_none() { - missing_blobs.push(BlobIdentifier { - block_root: import_data.block_root, - index: i as u64, - }) - } - } + let cache = occupied_entry.get_mut(); - let _ = cache - .executed_block - .insert(AvailabilityPendingExecutedBlock::new( - block, - import_data, - payload_verification_outcome, - )); + let missing_blob_ids = executed_block + .get_filtered_blob_ids(|index| cache.verified_blobs.get(index).is_none()); + + let _ = cache.executed_block.insert(executed_block); - Ok(Availability::PendingBlobs(missing_blobs)) + Ok(Availability::PendingBlobs(missing_blob_ids)) } } @@ -397,6 +403,13 @@ impl DataAvailabilityChecker { self.data_availability_boundary() .map_or(false, |da_epoch| block_epoch >= da_epoch) } + + pub fn prune_rpc_blob_cache(&self, blob_ids: &[BlobIdentifier]) { + let mut guard = self.rpc_blob_cache.write(); + for id in blob_ids { + guard.remove(id); + } + } } pub enum BlobRequirements { From da7fab51882775cac60484744cb8f35c63710a77 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 28 Mar 2023 11:47:30 -0400 Subject: [PATCH 34/96] Ef test version update (#4142) * Revert "revert change to ef_tests" This reverts commit 1093ba1a2717c21496de171db01669ae0e6d40dd. * Revert "Revert "Use consensus-spec-tests `v1.3.0-rc.3` (#4021)"" This reverts commit 20be7024e149e41c928fe7f0426724a68e5baff1. * update tests --- testing/ef_tests/Makefile | 2 +- testing/ef_tests/check_all_files_accessed.py | 5 ++++- .../src/cases/merkle_proof_validity.rs | 5 ----- testing/ef_tests/src/handler.rs | 19 +++++++++---------- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index e663ead1ab5..38b863b9fb9 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,4 +1,4 @@ -TESTS_TAG :=v1.3.0-rc.5 +TESTS_TAG := v1.3.0-rc.5 TESTS = general minimal mainnet TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index e1944aaf45e..60db5125fc0 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -57,9 +57,11 @@ "tests/general/deneb/kzg" ] + def normalize_path(path): return path.split("consensus-spec-tests/")[1] + # Determine the list of filenames which were accessed during tests. passed = set() for line in open(accessed_files_filename, 'r').readlines(): @@ -92,4 +94,5 @@ def normalize_path(path): # Exit with an error if there were any files missed. assert len(missed) == 0, "{} missed files".format(len(missed)) -print("Accessed {} files ({} intentionally excluded)".format(accessed_files, excluded_files)) +print("Accessed {} files ({} intentionally excluded)".format( + accessed_files, excluded_files)) diff --git a/testing/ef_tests/src/cases/merkle_proof_validity.rs b/testing/ef_tests/src/cases/merkle_proof_validity.rs index cdcdaaf9b36..c180774bb64 100644 --- a/testing/ef_tests/src/cases/merkle_proof_validity.rs +++ b/testing/ef_tests/src/cases/merkle_proof_validity.rs @@ -28,11 +28,6 @@ pub struct MerkleProofValidity { impl LoadCase for MerkleProofValidity { fn load_from_dir(path: &Path, fork_name: ForkName) -> Result { - //FIXME(sean) - if path.ends_with("execution_merkle_proof") { - return Err(Error::SkippedKnownFailure); - } - let spec = &testing_spec::(fork_name); let state = ssz_decode_state(&path.join("object.ssz_snappy"), spec)?; let merkle_proof = yaml_decode_file(&path.join("proof.yaml"))?; diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index 23fb20cf201..e6ca6aeaa04 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -1,6 +1,6 @@ use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation}; +use crate::type_name; use crate::type_name::TypeName; -use crate::{type_name, Error}; use derivative::Derivative; use std::fs::{self, DirEntry}; use std::marker::PhantomData; @@ -57,17 +57,11 @@ pub trait Handler { .filter_map(as_directory) .flat_map(|suite| fs::read_dir(suite.path()).expect("suite dir exists")) .filter_map(as_directory) - .filter_map(|test_case_dir| { + .map(|test_case_dir| { let path = test_case_dir.path(); - let case_result = Self::Case::load_from_dir(&path, fork_name); - - if let Err(Error::SkippedKnownFailure) = case_result.as_ref() { - return None; - } - - let case = case_result.expect("test should load"); - Some((path, case)) + let case = Self::Case::load_from_dir(&path, fork_name).expect("test should load"); + (path, case) }) .collect(); @@ -664,6 +658,11 @@ impl Handler for MerkleProofValidityHandler { fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool { fork_name != ForkName::Base + // Test is skipped due to some changes in the Capella light client + // spec. + // + // https://github.com/sigp/lighthouse/issues/4022 + && fork_name != ForkName::Capella && fork_name != ForkName::Deneb } } From d24e5cc22a974e13b795cd8967546749a4afe9e2 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 28 Mar 2023 12:49:19 -0400 Subject: [PATCH 35/96] clean up blobs by range response (#4137) --- .../network/src/beacon_processor/mod.rs | 1 - .../beacon_processor/worker/rpc_methods.rs | 80 ++++++++----------- 2 files changed, 33 insertions(+), 48 deletions(-) diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/network/src/beacon_processor/mod.rs index f58f2813515..256aa5e6072 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/network/src/beacon_processor/mod.rs @@ -1971,7 +1971,6 @@ impl BeaconProcessor { request, } => task_spawner.spawn_blocking_with_manual_send_idle(move |send_idle_on_drop| { worker.handle_blobs_by_range_request( - sub_executor, send_idle_on_drop, peer_id, request_id, diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index e00834872f9..ffdaa02dcfd 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -682,7 +682,6 @@ impl Worker { /// Handle a `BlobsByRange` request from the peer. pub fn handle_blobs_by_range_request( self, - _executor: TaskExecutor, send_on_drop: SendOnDrop, peer_id: PeerId, request_id: PeerRequestId, @@ -704,13 +703,15 @@ impl Worker { ); } - let data_availability_boundary = match self.chain.data_availability_boundary() { - Some(boundary) => boundary, + let request_start_slot = Slot::from(req.start_slot); + + let data_availability_boundary_slot = match self.chain.data_availability_boundary() { + Some(boundary) => boundary.start_slot(T::EthSpec::slots_per_epoch()), None => { debug!(self.log, "Deneb fork is disabled"); self.send_error_response( peer_id, - RPCResponseErrorCode::ServerError, + RPCResponseErrorCode::InvalidRequest, "Deneb fork is disabled".into(), request_id, ); @@ -718,47 +719,40 @@ impl Worker { } }; - let start_slot = Slot::from(req.start_slot); - let start_epoch = start_slot.epoch(T::EthSpec::slots_per_epoch()); - - // If the peer requests data from beyond the data availability boundary we altruistically - // cap to the right time range. - let serve_blobs_from_slot = if start_epoch < data_availability_boundary { - // Attempt to serve from the earliest block in our database, falling back to the data - // availability boundary - let oldest_blob_slot = - self.chain.store.get_blob_info().oldest_blob_slot.unwrap_or( - data_availability_boundary.start_slot(T::EthSpec::slots_per_epoch()), - ); - + let oldest_blob_slot = self + .chain + .store + .get_blob_info() + .oldest_blob_slot + .unwrap_or(data_availability_boundary_slot); + if request_start_slot < oldest_blob_slot { debug!( self.log, - "Range request start slot is older than data availability boundary"; - "requested_slot" => req.start_slot, - "oldest_known_slot" => oldest_blob_slot, - "data_availability_boundary" => data_availability_boundary + "Range request start slot is older than data availability boundary."; + "requested_slot" => request_start_slot, + "oldest_blob_slot" => oldest_blob_slot, + "data_availability_boundary" => data_availability_boundary_slot ); - // Check if the request is entirely out of the data availability period. The - // `oldest_blob_slot` is the oldest slot in the database, so includes a margin of error - // controlled by our prune margin. - let end_request_slot = start_slot + req.count; - if oldest_blob_slot < end_request_slot { - return self.send_error_response( + return if data_availability_boundary_slot < oldest_blob_slot { + self.send_error_response( + peer_id, + RPCResponseErrorCode::ResourceUnavailable, + "blobs pruned within boundary".into(), + request_id, + ) + } else { + self.send_error_response( peer_id, RPCResponseErrorCode::InvalidRequest, - "Request outside of data availability period".into(), + "Req outside availability period".into(), request_id, - ); - } - std::cmp::max(oldest_blob_slot, start_slot) - } else { - start_slot - }; + ) + }; + } - // If the peer requests data from beyond the data availability boundary we altruistically cap to the right time range let forwards_block_root_iter = - match self.chain.forwards_iter_block_roots(serve_blobs_from_slot) { + match self.chain.forwards_iter_block_roots(request_start_slot) { Ok(iter) => iter, Err(BeaconChainError::HistoricalBlockError( HistoricalBlockError::BlockOutOfRange { @@ -849,21 +843,13 @@ impl Worker { } } Ok(None) => { - error!( + debug!( self.log, - "No blobs or block in the store for block root"; + "No blobs in the store for block root"; "request" => ?req, "peer" => %peer_id, "block_root" => ?root ); - self.send_error_response( - peer_id, - RPCResponseErrorCode::ServerError, - "Database inconsistency".into(), - request_id, - ); - send_response = false; - break; } Err(BeaconChainError::BlobsUnavailable) => { error!( @@ -885,7 +871,7 @@ impl Worker { Err(e) => { error!( self.log, - "Error fetching blinded block for block root"; + "Error fetching blobs block root"; "request" => ?req, "peer" => %peer_id, "block_root" => ?root, From deec9c51baee7d88e27d3342a9472a4d7fdc16a7 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 28 Mar 2023 12:49:32 -0400 Subject: [PATCH 36/96] clean up blob by root response (#4136) --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- .../src/data_availability_checker.rs | 5 + .../beacon_chain/src/early_attester_cache.rs | 1 - beacon_node/beacon_chain/src/errors.rs | 3 - .../network/src/beacon_processor/mod.rs | 8 +- .../beacon_processor/worker/rpc_methods.rs | 214 ++++++------------ 6 files changed, 71 insertions(+), 162 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 706db639606..1afff4a9587 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1023,7 +1023,7 @@ impl BeaconChain { ) } - pub async fn get_blobs_checking_early_attester_cache( + pub fn get_blobs_checking_early_attester_cache( &self, block_root: &Hash256, ) -> Result>, Error> { diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 3046c8b396a..0cf26f12f58 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -112,6 +112,11 @@ impl DataAvailabilityChecker { } } + /// Get a blob from the RPC cache. + pub fn get_blob(&self, blob_id: &BlobIdentifier) -> Option>> { + self.rpc_blob_cache.read().get(blob_id).cloned() + } + /// This first validate the KZG commitments included in the blob sidecar. /// Check if we've cached other blobs for this block. If it completes a set and we also /// have a block cached, return the Availability variant triggering block import. diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index 082c2242c84..0aecbde1695 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -22,7 +22,6 @@ pub struct CacheItem { * Values used to make the block available. */ block: Arc>, - //TODO(sean) remove this and just use the da checker?' blobs: Option>, proto_block: ProtoBlock, } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 2935250faf4..9baa638f450 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -213,9 +213,6 @@ pub enum BeaconChainError { BlsToExecutionConflictsWithPool, InconsistentFork(InconsistentFork), ProposerHeadForkChoiceError(fork_choice::Error), - BlobsUnavailable, - NoKzgCommitmentsFieldOnBlock, - BlobsOlderThanDataAvailabilityBoundary(Epoch), } easy_from_to!(SlotProcessingError, BeaconChainError); diff --git a/beacon_node/network/src/beacon_processor/mod.rs b/beacon_node/network/src/beacon_processor/mod.rs index 256aa5e6072..c26fe757276 100644 --- a/beacon_node/network/src/beacon_processor/mod.rs +++ b/beacon_node/network/src/beacon_processor/mod.rs @@ -1983,13 +1983,7 @@ impl BeaconProcessor { request_id, request, } => task_spawner.spawn_blocking_with_manual_send_idle(move |send_idle_on_drop| { - worker.handle_blobs_by_root_request( - sub_executor, - send_idle_on_drop, - peer_id, - request_id, - request, - ) + worker.handle_blobs_by_root_request(send_idle_on_drop, peer_id, request_id, request) }), /* diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index ffdaa02dcfd..c8667b3528f 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -218,150 +218,81 @@ impl Worker { /// Handle a `BlobsByRoot` request from the peer. pub fn handle_blobs_by_root_request( self, - executor: TaskExecutor, send_on_drop: SendOnDrop, peer_id: PeerId, request_id: PeerRequestId, request: BlobsByRootRequest, ) { - // TODO: this code is grossly adjusted to free the blobs. Needs love <3 - // Fetching blocks is async because it may have to hit the execution layer for payloads. - executor.spawn( - async move { - let requested_blobs = request.blob_ids.len(); - let mut send_blob_count = 0; - let mut send_response = true; - - let mut blob_list_results = HashMap::new(); - for BlobIdentifier{ block_root: root, index } in request.blob_ids.into_iter() { - let blob_list_result = match blob_list_results.entry(root) { - Entry::Vacant(entry) => { - entry.insert(self - .chain - .get_blobs_checking_early_attester_cache(&root) - .await) - } - Entry::Occupied(entry) => { - entry.into_mut() - } - }; + let requested_blobs = request.blob_ids.len(); + let mut send_blob_count = 0; + let send_response = true; + + let mut blob_list_results = HashMap::new(); + for id in request.blob_ids.into_iter() { + // First attempt to get the blobs from the RPC cache. + if let Some(blob) = self.chain.data_availability_checker.get_blob(&id) { + self.send_response(peer_id, Response::BlobsByRoot(Some(blob)), request_id); + send_blob_count += 1; + } else { + let BlobIdentifier { + block_root: root, + index, + } = id; + + let blob_list_result = match blob_list_results.entry(root) { + Entry::Vacant(entry) => { + entry.insert(self.chain.get_blobs_checking_early_attester_cache(&root)) + } + Entry::Occupied(entry) => entry.into_mut(), + }; - match blob_list_result.as_ref() { - Ok(Some(blobs_sidecar_list)) => { - for blob_sidecar in blobs_sidecar_list.iter() { - if blob_sidecar.index == index { - self.send_response( - peer_id, - Response::BlobsByRoot(Some(blob_sidecar.clone())), - request_id, - ); - send_blob_count += 1; - break; - } - } - } - Ok(None) => { - debug!( - self.log, - "Peer requested unknown block and blobs"; - "peer" => %peer_id, - "request_root" => ?root - ); - } - Err(BeaconChainError::BlobsUnavailable) => { - error!( - self.log, - "No blobs in the store for block root"; - "peer" => %peer_id, - "block_root" => ?root - ); - self.send_error_response( - peer_id, - RPCResponseErrorCode::BlobsNotFoundForBlock, - "Blobs not found for block root".into(), - request_id, - ); - send_response = false; - break; - } - Err(BeaconChainError::NoKzgCommitmentsFieldOnBlock) => { - debug!( - self.log, - "Peer requested blobs for a pre-deneb block"; - "peer" => %peer_id, - "block_root" => ?root, - ); - self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Failed reading field kzg_commitments from block".into(), - request_id, - ); - send_response = false; - break; - } - Err(BeaconChainError::BlobsOlderThanDataAvailabilityBoundary(block_epoch)) => { - debug!( - self.log, - "Peer requested block and blobs older than the data availability \ - boundary for ByRoot request, no blob found"; - "peer" => %peer_id, - "request_root" => ?root, - "block_epoch" => ?block_epoch, + match blob_list_result.as_ref() { + Ok(Some(blobs_sidecar_list)) => { + 'inner: for blob_sidecar in blobs_sidecar_list.iter() { + if blob_sidecar.index == index { + self.send_response( + peer_id, + Response::BlobsByRoot(Some(blob_sidecar.clone())), + request_id, ); - self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Blobs older than data availability boundary".into(), - request_id, - ); - send_response = false; - break; - } - Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => { - debug!( - self.log, - "Failed to fetch execution payload for block and blobs by root request"; - "block_root" => ?root, - "reason" => "execution layer not synced", - ); - // send the stream terminator - self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Execution layer not synced".into(), - request_id, - ); - send_response = false; - break; - } - Err(e) => { - debug!( - self.log, - "Error fetching block for peer"; - "peer" => %peer_id, - "request_root" => ?root, - "error" => ?e, - ); + send_blob_count += 1; + break 'inner; + } } } + Ok(None) => { + debug!( + self.log, + "Peer requested unknown blobs"; + "peer" => %peer_id, + "request_root" => ?root + ); + } + Err(e) => { + debug!( + self.log, + "Error fetching blob for peer"; + "peer" => %peer_id, + "request_root" => ?root, + "error" => ?e, + ); + } } - debug!( - self.log, - "Received BlobsByRoot Request"; - "peer" => %peer_id, - "requested" => requested_blobs, - "returned" => send_blob_count - ); + } + } + debug!( + self.log, + "Received BlobsByRoot Request"; + "peer" => %peer_id, + "requested" => requested_blobs, + "returned" => send_blob_count + ); - // send stream termination - if send_response { - self.send_response(peer_id, Response::BlobsByRoot(None), request_id); - } - drop(send_on_drop); - }, - "load_blobs_by_root_blocks", - ) + // send stream termination + if send_response { + self.send_response(peer_id, Response::BlobsByRoot(None), request_id); + } + drop(send_on_drop); } /// Handle a `BlocksByRoot` request from the peer. @@ -851,23 +782,6 @@ impl Worker { "block_root" => ?root ); } - Err(BeaconChainError::BlobsUnavailable) => { - error!( - self.log, - "No blobs in the store for block root"; - "request" => ?req, - "peer" => %peer_id, - "block_root" => ?root - ); - self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Blobs unavailable".into(), - request_id, - ); - send_response = false; - break; - } Err(e) => { error!( self.log, From ffefd201374a0870c41fbf336aa290548f82ddf2 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Mon, 3 Apr 2023 15:07:11 +0530 Subject: [PATCH 37/96] Block processing cleanup (#4153) * Implements Ord for BlobSidecar based on index * Use BTreeMap for gossip cache to maintain blob order by index * fmt * Another panic fix --- .../beacon_chain/src/blob_verification.rs | 29 ++++- .../beacon_chain/src/block_verification.rs | 6 +- .../src/data_availability_checker.rs | 108 ++++++++++++------ .../lighthouse_network/src/types/pubsub.rs | 12 +- consensus/types/src/blob_sidecar.rs | 26 ++++- 5 files changed, 132 insertions(+), 49 deletions(-) diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 48ad45e83b3..bab8646e914 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -263,11 +263,26 @@ pub fn validate_blob_sidecar_for_gossip( }) } -#[derive(Debug, Clone)] +/// Wrapper over a `BlobSidecar` for which we have completed kzg verification. +/// i.e. `verify_blob_kzg_proof(blob, commitment, proof) == true`. +#[derive(Debug, Derivative, Clone)] +#[derivative(PartialEq, Eq)] pub struct KzgVerifiedBlob { blob: Arc>, } +impl PartialOrd for KzgVerifiedBlob { + fn partial_cmp(&self, other: &Self) -> Option { + self.blob.partial_cmp(&other.blob) + } +} + +impl Ord for KzgVerifiedBlob { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.blob.cmp(&other.blob) + } +} + impl KzgVerifiedBlob { pub fn to_blob(self) -> Arc> { self.blob @@ -284,8 +299,14 @@ impl KzgVerifiedBlob { pub fn block_root(&self) -> Hash256 { self.blob.block_root } + pub fn blob_index(&self) -> u64 { + self.blob.index + } } +/// Complete kzg verification for a `GossipVerifiedBlob`. +/// +/// Returns an error if the kzg verification check fails. pub fn verify_kzg_for_blob( blob: GossipVerifiedBlob, kzg: &Kzg, @@ -305,6 +326,11 @@ pub fn verify_kzg_for_blob( } } +/// Complete kzg verification for a list of `BlobSidecar`s. +/// Returns an error if any of the `BlobSidecar`s fails kzg verification. +/// +/// Note: This function should be preferred over calling `verify_kzg_for_blob` +/// in a loop since this function kzg verifies a list of blobs more efficiently. pub fn verify_kzg_for_blob_list( blob_list: BlobSidecarList, kzg: &Kzg, @@ -344,6 +370,7 @@ pub enum MaybeAvailableBlock { AvailabilityPending(AvailabilityPendingBlock), } +/// Trait for common block operations. pub trait AsBlock { fn slot(&self) -> Slot; fn epoch(&self) -> Epoch; diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 7c137146865..36cc7233193 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -784,14 +784,14 @@ impl AvailabilityPendingExecutedBlock { self.get_filtered_blob_ids(|_| true) } - pub fn get_filtered_blob_ids(&self, filter: impl Fn(usize) -> bool) -> Vec { + pub fn get_filtered_blob_ids(&self, filter: impl Fn(u64) -> bool) -> Vec { let num_blobs_expected = self.num_blobs_expected(); let mut blob_ids = Vec::with_capacity(num_blobs_expected); - for i in 0..num_blobs_expected { + for i in 0..num_blobs_expected as u64 { if filter(i) { blob_ids.push(BlobIdentifier { block_root: self.import_data.block_root, - index: i as u64, + index: i, }); } } diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 0cf26f12f58..f2af8cdf892 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -11,7 +11,7 @@ use slot_clock::SlotClock; use ssz_types::{Error, VariableList}; use state_processing::per_block_processing::deneb::deneb::verify_kzg_commitments_against_transactions; use std::collections::hash_map::{Entry, OccupiedEntry}; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::{BlobIdentifier, BlobSidecar}; @@ -54,37 +54,51 @@ impl From for AvailabilityCheckError { /// - blocks that have been fully verified and only require a data availability check pub struct DataAvailabilityChecker { rpc_blob_cache: RwLock>>>, - gossip_blob_cache: Mutex>>, + gossip_availability_cache: Mutex>>, slot_clock: S, kzg: Option>, spec: ChainSpec, } -struct GossipBlobCache { - verified_blobs: Vec>, +/// Caches partially available blobs and execution verified blocks corresponding +/// to a given `block_hash` that are received over gossip. +/// +/// The blobs are all gossip and kzg verified. +/// The block has completed all verifications except the availability check. +struct GossipAvailabilityCache { + /// We use a `BTreeMap` here to maintain the order of `BlobSidecar`s based on index. + verified_blobs: BTreeMap>, executed_block: Option>, } -impl GossipBlobCache { +impl GossipAvailabilityCache { fn new_from_blob(blob: KzgVerifiedBlob) -> Self { + let mut verified_blobs = BTreeMap::new(); + verified_blobs.insert(blob.blob_index(), blob); Self { - verified_blobs: vec![blob], + verified_blobs, executed_block: None, } } fn new_from_block(block: AvailabilityPendingExecutedBlock) -> Self { Self { - verified_blobs: vec![], + verified_blobs: BTreeMap::new(), executed_block: Some(block), } } + /// Returns `true` if the cache has all blobs corresponding to the + /// kzg commitments in the block. fn has_all_blobs(&self, block: &AvailabilityPendingExecutedBlock) -> bool { self.verified_blobs.len() == block.num_blobs_expected() } } +/// This type is returned after adding a block / blob to the `DataAvailabilityChecker`. +/// +/// Indicates if the block is fully `Available` or if we need blobs or blocks +/// to "complete" the requirements for an `AvailableBlock`. pub enum Availability { PendingBlobs(Vec), PendingBlock(Hash256), @@ -92,6 +106,8 @@ pub enum Availability { } impl Availability { + /// Returns all the blob identifiers associated with an `AvailableBlock`. + /// Returns `None` if avaiability hasn't been fully satisfied yet. pub fn get_available_blob_ids(&self) -> Option> { if let Self::Available(block) = self { Some(block.get_all_blob_ids()) @@ -105,7 +121,7 @@ impl DataAvailabilityChecker { pub fn new(slot_clock: S, kzg: Option>, spec: ChainSpec) -> Self { Self { rpc_blob_cache: <_>::default(), - gossip_blob_cache: <_>::default(), + gossip_availability_cache: <_>::default(), slot_clock, kzg, spec, @@ -117,9 +133,9 @@ impl DataAvailabilityChecker { self.rpc_blob_cache.read().get(blob_id).cloned() } - /// This first validate the KZG commitments included in the blob sidecar. + /// This first validates the KZG commitments included in the blob sidecar. /// Check if we've cached other blobs for this block. If it completes a set and we also - /// have a block cached, return the Availability variant triggering block import. + /// have a block cached, return the `Availability` variant triggering block import. /// Otherwise cache the blob sidecar. /// /// This should only accept gossip verified blobs, so we should not have to worry about dupes. @@ -138,7 +154,7 @@ impl DataAvailabilityChecker { let blob = kzg_verified_blob.clone_blob(); - let mut blob_cache = self.gossip_blob_cache.lock(); + let mut blob_cache = self.gossip_availability_cache.lock(); // Gossip cache. let availability = match blob_cache.entry(blob.block_root) { @@ -149,7 +165,7 @@ impl DataAvailabilityChecker { cache .verified_blobs - .insert(blob.index as usize, kzg_verified_blob); + .insert(kzg_verified_blob.blob_index(), kzg_verified_blob); if let Some(executed_block) = cache.executed_block.take() { self.check_block_availability_maybe_cache(occupied_entry, executed_block)? @@ -159,7 +175,7 @@ impl DataAvailabilityChecker { } Entry::Vacant(vacant_entry) => { let block_root = kzg_verified_blob.block_root(); - vacant_entry.insert(GossipBlobCache::new_from_blob(kzg_verified_blob)); + vacant_entry.insert(GossipAvailabilityCache::new_from_blob(kzg_verified_blob)); Availability::PendingBlock(block_root) } }; @@ -181,7 +197,7 @@ impl DataAvailabilityChecker { &self, executed_block: AvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { - let mut guard = self.gossip_blob_cache.lock(); + let mut guard = self.gossip_availability_cache.lock(); let entry = guard.entry(executed_block.import_data.block_root); let availability = match entry { @@ -190,7 +206,7 @@ impl DataAvailabilityChecker { } Entry::Vacant(vacant_entry) => { let all_blob_ids = executed_block.get_all_blob_ids(); - vacant_entry.insert(GossipBlobCache::new_from_block(executed_block)); + vacant_entry.insert(GossipAvailabilityCache::new_from_block(executed_block)); Availability::PendingBlobs(all_blob_ids) } }; @@ -204,9 +220,16 @@ impl DataAvailabilityChecker { Ok(availability) } + /// Checks if the provided `executed_block` contains all required blobs to be considered an + /// `AvailableBlock` based on blobs that are cached. + /// + /// Returns an error if there was an error when matching the block commitments against blob commitments. + /// + /// Returns `Ok(Availability::Available(_))` if all blobs for the block are present in cache. + /// Returns `Ok(Availability::PendingBlobs(_))` if all corresponding blobs have not been received in the cache. fn check_block_availability_maybe_cache( &self, - mut occupied_entry: OccupiedEntry>, + mut occupied_entry: OccupiedEntry>, executed_block: AvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { if occupied_entry.get().has_all_blobs(&executed_block) { @@ -216,9 +239,13 @@ impl DataAvailabilityChecker { payload_verification_outcome, } = executed_block; - let cache = occupied_entry.remove(); + let GossipAvailabilityCache { + verified_blobs, + executed_block: _, + } = occupied_entry.remove(); + let verified_blobs = verified_blobs.into_values().collect(); - let available_block = self.make_available(block, cache.verified_blobs)?; + let available_block = self.make_available(block, verified_blobs)?; Ok(Availability::Available(Box::new( AvailableExecutedBlock::new( available_block, @@ -227,18 +254,18 @@ impl DataAvailabilityChecker { ), ))) } else { - let cache = occupied_entry.get_mut(); + let cached_entry = occupied_entry.get_mut(); let missing_blob_ids = executed_block - .get_filtered_blob_ids(|index| cache.verified_blobs.get(index).is_none()); + .get_filtered_blob_ids(|index| cached_entry.verified_blobs.get(&index).is_none()); - let _ = cache.executed_block.insert(executed_block); + let _ = cached_entry.executed_block.insert(executed_block); Ok(Availability::PendingBlobs(missing_blob_ids)) } } - /// Checks if a block is available, returns a MaybeAvailableBlock enum that may include the fully + /// Checks if a block is available, returns a `MaybeAvailableBlock` that may include the fully /// available block. pub fn check_availability( &self, @@ -321,11 +348,11 @@ impl DataAvailabilityChecker { /// Verifies an AvailabilityPendingBlock against a set of KZG verified blobs. /// This does not check whether a block *should* have blobs, these checks should must have been - /// completed when producing the AvailabilityPendingBlock. + /// completed when producing the `AvailabilityPendingBlock`. pub fn make_available( &self, block: AvailabilityPendingBlock, - blobs: KzgVerifiedBlobList, + blobs: Vec>, ) -> Result, AvailabilityCheckError> { let block_kzg_commitments = block.kzg_commitments()?; if blobs.len() != block_kzg_commitments.len() { @@ -428,6 +455,12 @@ pub enum BlobRequirements { PreDeneb, } +/// A wrapper over a `SignedBeaconBlock` where we have not verified availability of +/// corresponding `BlobSidecar`s and hence, is not ready for import into fork choice. +/// +/// Note: This wrapper does not necessarily correspond to a pre-deneb block as a pre-deneb +/// block that is ready for import will be of type `AvailableBlock` with its `blobs` field +/// set to `VerifiedBlobs::PreDeneb`. #[derive(Clone, Debug, PartialEq)] pub struct AvailabilityPendingBlock { block: Arc>, @@ -452,6 +485,20 @@ impl AvailabilityPendingBlock { } } +#[derive(Clone, Debug, PartialEq)] +pub enum VerifiedBlobs { + /// These blobs are available. + Available(BlobSidecarList), + /// This block is from outside the data availability boundary so doesn't require + /// a data availability check. + NotRequired, + /// The block's `kzg_commitments` field is empty so it does not contain any blobs. + EmptyBlobs, + /// This is a block prior to the 4844 fork, so doesn't require any blobs + PreDeneb, +} + +/// A fully available block that is ready to be imported into fork choice. #[derive(Clone, Debug, PartialEq)] pub struct AvailableBlock { block: Arc>, @@ -473,19 +520,6 @@ impl AvailableBlock { } } -#[derive(Clone, Debug, PartialEq)] -pub enum VerifiedBlobs { - /// These blobs are available. - Available(BlobSidecarList), - /// This block is from outside the data availability boundary so doesn't require - /// a data availability check. - NotRequired, - /// The block's `kzg_commitments` field is empty so it does not contain any blobs. - EmptyBlobs, - /// This is a block prior to the 4844 fork, so doesn't require any blobs - PreDeneb, -} - impl AsBlock for AvailableBlock { fn slot(&self) -> Slot { self.block.slot() diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 49cd8860b33..c69610cdb0b 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -12,7 +12,7 @@ use types::{ Attestation, AttesterSlashing, EthSpec, ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, - SignedBeaconBlockMerge, SignedBlobSidecar, SignedBlsToExecutionChange, + SignedBeaconBlockDeneb, SignedBeaconBlockMerge, SignedBlobSidecar, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; @@ -184,16 +184,14 @@ impl PubsubMessage { SignedBeaconBlockMerge::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, ), - Some(ForkName::Deneb) => { - return Err( - "beacon_block topic is not used from deneb fork onwards" - .to_string(), - ) - } Some(ForkName::Capella) => SignedBeaconBlock::::Capella( SignedBeaconBlockCapella::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, ), + Some(ForkName::Deneb) => SignedBeaconBlock::::Deneb( + SignedBeaconBlockDeneb::from_ssz_bytes(data) + .map_err(|e| format!("{:?}", e))?, + ), None => { return Err(format!( "Unknown gossipsub fork digest: {:?}", diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index ce6d7e0e611..fde54bc721c 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -17,6 +17,18 @@ pub struct BlobIdentifier { pub index: u64, } +impl PartialOrd for BlobIdentifier { + fn partial_cmp(&self, other: &Self) -> Option { + self.index.partial_cmp(&other.index) + } +} + +impl Ord for BlobIdentifier { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.index.cmp(&other.index) + } +} + #[derive( Debug, Clone, @@ -32,7 +44,7 @@ pub struct BlobIdentifier { )] #[serde(bound = "T: EthSpec")] #[arbitrary(bound = "T: EthSpec")] -#[derivative(PartialEq, Hash(bound = "T: EthSpec"))] +#[derivative(PartialEq, Eq, Hash(bound = "T: EthSpec"))] pub struct BlobSidecar { pub block_root: Hash256, #[serde(with = "eth2_serde_utils::quoted_u64")] @@ -47,6 +59,18 @@ pub struct BlobSidecar { pub kzg_proof: KzgProof, } +impl PartialOrd for BlobSidecar { + fn partial_cmp(&self, other: &Self) -> Option { + self.index.partial_cmp(&other.index) + } +} + +impl Ord for BlobSidecar { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.index.cmp(&other.index) + } +} + pub type BlobSidecarList = VariableList>, ::MaxBlobsPerBlock>; pub type Blobs = VariableList, ::MaxExtraDataBytes>; From 3a213176007bd3e9db31468c09be7206b82a1b10 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Tue, 4 Apr 2023 08:50:35 -0500 Subject: [PATCH 38/96] Unified Availability Cache into One (#4161) * Unified Availability Cache into One * Update beacon_node/beacon_chain/src/data_availability_checker.rs Co-authored-by: Jimmy Chen --------- Co-authored-by: realbigsean Co-authored-by: Jimmy Chen --- .../src/data_availability_checker.rs | 131 ++++++++++-------- 1 file changed, 71 insertions(+), 60 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index f2af8cdf892..4c191695b20 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -6,12 +6,12 @@ use crate::block_verification::{AvailabilityPendingExecutedBlock, AvailableExecu use kzg::Error as KzgError; use kzg::Kzg; -use parking_lot::{Mutex, RwLock}; +use parking_lot::RwLock; use slot_clock::SlotClock; -use ssz_types::{Error, VariableList}; +use ssz_types::{Error, FixedVector, VariableList}; use state_processing::per_block_processing::deneb::deneb::verify_kzg_commitments_against_transactions; use std::collections::hash_map::{Entry, OccupiedEntry}; -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::sync::Arc; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::{BlobIdentifier, BlobSidecar}; @@ -53,8 +53,7 @@ impl From for AvailabilityCheckError { /// have not been verified against blobs /// - blocks that have been fully verified and only require a data availability check pub struct DataAvailabilityChecker { - rpc_blob_cache: RwLock>>>, - gossip_availability_cache: Mutex>>, + availability_cache: RwLock>>, slot_clock: S, kzg: Option>, spec: ChainSpec, @@ -65,16 +64,20 @@ pub struct DataAvailabilityChecker { /// /// The blobs are all gossip and kzg verified. /// The block has completed all verifications except the availability check. -struct GossipAvailabilityCache { +struct ReceivedComponents { /// We use a `BTreeMap` here to maintain the order of `BlobSidecar`s based on index. - verified_blobs: BTreeMap>, + verified_blobs: FixedVector>, T::MaxBlobsPerBlock>, executed_block: Option>, } -impl GossipAvailabilityCache { +impl ReceivedComponents { fn new_from_blob(blob: KzgVerifiedBlob) -> Self { - let mut verified_blobs = BTreeMap::new(); - verified_blobs.insert(blob.blob_index(), blob); + let mut verified_blobs = FixedVector::<_, _>::default(); + // TODO: verify that we've already ensured the blob index < T::MaxBlobsPerBlock + if let Some(mut_maybe_blob) = verified_blobs.get_mut(blob.blob_index() as usize) { + *mut_maybe_blob = Some(blob); + } + Self { verified_blobs, executed_block: None, @@ -83,7 +86,7 @@ impl GossipAvailabilityCache { fn new_from_block(block: AvailabilityPendingExecutedBlock) -> Self { Self { - verified_blobs: BTreeMap::new(), + verified_blobs: <_>::default(), executed_block: Some(block), } } @@ -91,7 +94,17 @@ impl GossipAvailabilityCache { /// Returns `true` if the cache has all blobs corresponding to the /// kzg commitments in the block. fn has_all_blobs(&self, block: &AvailabilityPendingExecutedBlock) -> bool { - self.verified_blobs.len() == block.num_blobs_expected() + for i in 0..block.num_blobs_expected() { + if self + .verified_blobs + .get(i) + .map(|maybe_blob| maybe_blob.is_none()) + .unwrap_or(true) + { + return false; + } + } + true } } @@ -120,17 +133,22 @@ impl Availability { impl DataAvailabilityChecker { pub fn new(slot_clock: S, kzg: Option>, spec: ChainSpec) -> Self { Self { - rpc_blob_cache: <_>::default(), - gossip_availability_cache: <_>::default(), + availability_cache: <_>::default(), slot_clock, kzg, spec, } } - /// Get a blob from the RPC cache. + /// Get a blob from the availability cache. pub fn get_blob(&self, blob_id: &BlobIdentifier) -> Option>> { - self.rpc_blob_cache.read().get(blob_id).cloned() + self.availability_cache + .read() + .get(&blob_id.block_root)? + .verified_blobs + .get(blob_id.index as usize)? + .as_ref() + .map(|kzg_verified_blob| kzg_verified_blob.clone_blob()) } /// This first validates the KZG commitments included in the blob sidecar. @@ -152,22 +170,24 @@ impl DataAvailabilityChecker { return Err(AvailabilityCheckError::KzgNotInitialized); }; - let blob = kzg_verified_blob.clone_blob(); - - let mut blob_cache = self.gossip_availability_cache.lock(); - - // Gossip cache. - let availability = match blob_cache.entry(blob.block_root) { + let availability = match self + .availability_cache + .write() + .entry(kzg_verified_blob.block_root()) + { Entry::Occupied(mut occupied_entry) => { // All blobs reaching this cache should be gossip verified and gossip verification // should filter duplicates, as well as validate indices. - let cache = occupied_entry.get_mut(); + let received_components = occupied_entry.get_mut(); - cache + if let Some(maybe_verified_blob) = received_components .verified_blobs - .insert(kzg_verified_blob.blob_index(), kzg_verified_blob); + .get_mut(kzg_verified_blob.blob_index() as usize) + { + *maybe_verified_blob = Some(kzg_verified_blob) + } - if let Some(executed_block) = cache.executed_block.take() { + if let Some(executed_block) = received_components.executed_block.take() { self.check_block_availability_maybe_cache(occupied_entry, executed_block)? } else { Availability::PendingBlock(block_root) @@ -175,19 +195,11 @@ impl DataAvailabilityChecker { } Entry::Vacant(vacant_entry) => { let block_root = kzg_verified_blob.block_root(); - vacant_entry.insert(GossipAvailabilityCache::new_from_blob(kzg_verified_blob)); + vacant_entry.insert(ReceivedComponents::new_from_blob(kzg_verified_blob)); Availability::PendingBlock(block_root) } }; - drop(blob_cache); - - if let Some(blob_ids) = availability.get_available_blob_ids() { - self.prune_rpc_blob_cache(&blob_ids); - } else { - self.rpc_blob_cache.write().insert(blob.id(), blob.clone()); - } - Ok(availability) } @@ -197,26 +209,21 @@ impl DataAvailabilityChecker { &self, executed_block: AvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { - let mut guard = self.gossip_availability_cache.lock(); - let entry = guard.entry(executed_block.import_data.block_root); - - let availability = match entry { + let availability = match self + .availability_cache + .write() + .entry(executed_block.import_data.block_root) + { Entry::Occupied(occupied_entry) => { self.check_block_availability_maybe_cache(occupied_entry, executed_block)? } Entry::Vacant(vacant_entry) => { let all_blob_ids = executed_block.get_all_blob_ids(); - vacant_entry.insert(GossipAvailabilityCache::new_from_block(executed_block)); + vacant_entry.insert(ReceivedComponents::new_from_block(executed_block)); Availability::PendingBlobs(all_blob_ids) } }; - drop(guard); - - if let Some(blob_ids) = availability.get_available_blob_ids() { - self.prune_rpc_blob_cache(&blob_ids); - } - Ok(availability) } @@ -229,21 +236,27 @@ impl DataAvailabilityChecker { /// Returns `Ok(Availability::PendingBlobs(_))` if all corresponding blobs have not been received in the cache. fn check_block_availability_maybe_cache( &self, - mut occupied_entry: OccupiedEntry>, + mut occupied_entry: OccupiedEntry>, executed_block: AvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { if occupied_entry.get().has_all_blobs(&executed_block) { + let num_blobs_expected = executed_block.num_blobs_expected(); let AvailabilityPendingExecutedBlock { block, import_data, payload_verification_outcome, } = executed_block; - let GossipAvailabilityCache { + let ReceivedComponents { verified_blobs, executed_block: _, } = occupied_entry.remove(); - let verified_blobs = verified_blobs.into_values().collect(); + + let verified_blobs = Vec::from(verified_blobs) + .into_iter() + .take(num_blobs_expected) + .map(|maybe_blob| maybe_blob.ok_or(AvailabilityCheckError::MissingBlobs)) + .collect::, _>>()?; let available_block = self.make_available(block, verified_blobs)?; Ok(Availability::Available(Box::new( @@ -254,12 +267,17 @@ impl DataAvailabilityChecker { ), ))) } else { - let cached_entry = occupied_entry.get_mut(); + let received_components = occupied_entry.get_mut(); - let missing_blob_ids = executed_block - .get_filtered_blob_ids(|index| cached_entry.verified_blobs.get(&index).is_none()); + let missing_blob_ids = executed_block.get_filtered_blob_ids(|index| { + received_components + .verified_blobs + .get(index as usize) + .map(|maybe_blob| maybe_blob.is_none()) + .unwrap_or(true) + }); - let _ = cached_entry.executed_block.insert(executed_block); + let _ = received_components.executed_block.insert(executed_block); Ok(Availability::PendingBlobs(missing_blob_ids)) } @@ -435,13 +453,6 @@ impl DataAvailabilityChecker { self.data_availability_boundary() .map_or(false, |da_epoch| block_epoch >= da_epoch) } - - pub fn prune_rpc_blob_cache(&self, blob_ids: &[BlobIdentifier]) { - let mut guard = self.rpc_blob_cache.write(); - for id in blob_ids { - guard.remove(id); - } - } } pub enum BlobRequirements { From 1b8225c76da2f3596e325c330b8f6d8a8346cb89 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Wed, 5 Apr 2023 22:13:39 +0530 Subject: [PATCH 39/96] Revert upgrade to tokio utils to reprocessing queue (#4167) --- Cargo.lock | 2 +- beacon_node/network/Cargo.toml | 2 +- .../work_reprocessing_queue.rs | 34 +++++++++++++++---- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37bdb4294b0..c2d36c7c85b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5232,7 +5232,7 @@ dependencies = [ "task_executor", "tokio", "tokio-stream", - "tokio-util 0.7.7", + "tokio-util 0.6.10", "types", ] diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index ea415c00558..d068a20079b 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -41,7 +41,7 @@ num_cpus = "1.13.0" lru_cache = { path = "../../common/lru_cache" } if-addrs = "0.6.4" strum = "0.24.0" -tokio-util = { version = "0.7.7", features = ["time"] } +tokio-util = { version = "0.6.3", features = ["time"] } derivative = "2.2.0" delay_map = "0.3.0" ethereum-types = { version = "0.14.1", optional = true } diff --git a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs b/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs index 4a565307991..2ec10439b3c 100644 --- a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs +++ b/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs @@ -21,7 +21,7 @@ use futures::task::Poll; use futures::{Stream, StreamExt}; use lighthouse_network::{MessageId, PeerId}; use logging::TimeLatch; -use slog::{debug, error, trace, warn, Logger}; +use slog::{crit, debug, error, trace, warn, Logger}; use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; use std::pin::Pin; @@ -29,6 +29,7 @@ use std::task::Context; use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, Receiver, Sender}; +use tokio::time::error::Error as TimeError; use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey}; use types::{ Attestation, EthSpec, Hash256, LightClientOptimisticUpdate, SignedAggregateAndProof, SubnetId, @@ -154,6 +155,8 @@ enum InboundEvent { ReadyAttestation(QueuedAttestationId), /// A light client update that is ready for re-processing. ReadyLightClientUpdate(QueuedLightClientUpdateId), + /// A `DelayQueue` returned an error. + DelayQueueError(TimeError, &'static str), /// A message sent to the `ReprocessQueue` Msg(ReprocessQueueMessage), } @@ -231,42 +234,54 @@ impl Stream for ReprocessQueue { // The sequential nature of blockchains means it is generally better to try and import all // existing blocks before new ones. match self.gossip_block_delay_queue.poll_expired(cx) { - Poll::Ready(Some(queued_block)) => { + Poll::Ready(Some(Ok(queued_block))) => { return Poll::Ready(Some(InboundEvent::ReadyGossipBlock( queued_block.into_inner(), ))); } + Poll::Ready(Some(Err(e))) => { + return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "gossip_block_queue"))); + } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), } match self.rpc_block_delay_queue.poll_expired(cx) { - Poll::Ready(Some(queued_block)) => { + Poll::Ready(Some(Ok(queued_block))) => { return Poll::Ready(Some(InboundEvent::ReadyRpcBlock(queued_block.into_inner()))); } + Poll::Ready(Some(Err(e))) => { + return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "rpc_block_queue"))); + } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), } match self.attestations_delay_queue.poll_expired(cx) { - Poll::Ready(Some(attestation_id)) => { + Poll::Ready(Some(Ok(attestation_id))) => { return Poll::Ready(Some(InboundEvent::ReadyAttestation( attestation_id.into_inner(), ))); } + Poll::Ready(Some(Err(e))) => { + return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "attestations_queue"))); + } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), } match self.lc_updates_delay_queue.poll_expired(cx) { - Poll::Ready(Some(lc_id)) => { + Poll::Ready(Some(Ok(lc_id))) => { return Poll::Ready(Some(InboundEvent::ReadyLightClientUpdate( lc_id.into_inner(), ))); } + Poll::Ready(Some(Err(e))) => { + return Poll::Ready(Some(InboundEvent::DelayQueueError(e, "lc_updates_queue"))); + } // `Poll::Ready(None)` means that there are no more entries in the delay queue and we // will continue to get this result until something else is added into the queue. Poll::Ready(None) | Poll::Pending => (), @@ -689,7 +704,14 @@ impl ReprocessQueue { ); } } - + InboundEvent::DelayQueueError(e, queue_name) => { + crit!( + log, + "Failed to poll queue"; + "queue" => queue_name, + "e" => ?e + ) + } InboundEvent::ReadyAttestation(queued_id) => { metrics::inc_counter( &metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_EXPIRED_ATTESTATIONS, From fca8559acc2a15125ef24b3d62481163924bac0a Mon Sep 17 00:00:00 2001 From: Divma <26765164+divagant-martian@users.noreply.github.com> Date: Mon, 10 Apr 2023 19:05:01 -0500 Subject: [PATCH 40/96] Update kzg to get windows going, expose blst features (#4177) * fmt * update kzg * use commit from main repo --- Cargo.lock | 28 +++++++++++++++++++++++-- beacon_node/http_api/src/lib.rs | 4 ++-- crypto/kzg/Cargo.toml | 2 +- crypto/kzg/src/lib.rs | 16 +++++++------- validator_client/src/validator_store.rs | 5 +++-- 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9868c7db2ee..9a0b97d2e03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -704,6 +704,27 @@ dependencies = [ "shlex", ] +[[package]] +name = "bindgen" +version = "0.64.0" +source = "git+https://github.com/rust-lang/rust-bindgen?rev=0de11f0a521611ac8738b7b01d19dddaf3899e66#0de11f0a521611ac8738b7b01d19dddaf3899e66" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.13", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -925,8 +946,11 @@ dependencies = [ [[package]] name = "c-kzg" version = "0.1.0" -source = "git+https://github.com/ethereum/c-kzg-4844?rev=549739fcb3aaec6fe5651e1912f05c604b45621b#549739fcb3aaec6fe5651e1912f05c604b45621b" +source = "git+https://github.com/ethereum/c-kzg-4844?rev=fd24cf8e1e2f09a96b4e62a595b4e49f046ce6cf#fd24cf8e1e2f09a96b4e62a595b4e49f046ce6cf" dependencies = [ + "bindgen 0.64.0", + "cc", + "glob", "hex", "libc", "serde", @@ -4816,7 +4840,7 @@ name = "mdbx-sys" version = "0.11.6-4" source = "git+https://github.com/sigp/libmdbx-rs?tag=v0.1.4#096da80a83d14343f8df833006483f48075cd135" dependencies = [ - "bindgen", + "bindgen 0.59.2", "cc", "cmake", "libc", diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 5d1838fa0e7..9f92ef2e986 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -32,8 +32,8 @@ use beacon_chain::{ pub use block_id::BlockId; use directory::DEFAULT_ROOT_DIR; use eth2::types::{ - self as api_types, EndpointVersion, SignedBlockContents, ForkChoice, ForkChoiceNode, SkipRandaoVerification, - ValidatorId, ValidatorStatus, + self as api_types, EndpointVersion, ForkChoice, ForkChoiceNode, SignedBlockContents, + SkipRandaoVerification, ValidatorId, ValidatorStatus, }; use lighthouse_network::{types::SyncState, EnrExt, NetworkGlobals, PeerId, PubsubMessage}; use lighthouse_version::version_with_platform; diff --git a/crypto/kzg/Cargo.toml b/crypto/kzg/Cargo.toml index 0645b3a2b93..5c4a499e97f 100644 --- a/crypto/kzg/Cargo.toml +++ b/crypto/kzg/Cargo.toml @@ -16,7 +16,7 @@ serde_derive = "1.0.116" eth2_serde_utils = "0.1.1" hex = "0.4.2" eth2_hashing = "0.3.0" -c-kzg = {git = "https://github.com/ethereum/c-kzg-4844", rev = "549739fcb3aaec6fe5651e1912f05c604b45621b" } +c-kzg = {git = "https://github.com/ethereum/c-kzg-4844", rev = "fd24cf8e1e2f09a96b4e62a595b4e49f046ce6cf" } arbitrary = { version = "1.0", features = ["derive"], optional = true } [features] diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index cd28c8529a9..5379d36ed0a 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -5,7 +5,7 @@ mod trusted_setup; pub use crate::{kzg_commitment::KzgCommitment, kzg_proof::KzgProof, trusted_setup::TrustedSetup}; use c_kzg::Bytes48; pub use c_kzg::{ - Blob, Error as CKzgError, KZGSettings, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT, + Blob, Error as CKzgError, KzgSettings, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT, FIELD_ELEMENTS_PER_BLOB, }; use std::path::PathBuf; @@ -21,7 +21,7 @@ pub enum Error { /// A wrapper over a kzg library that holds the trusted setup parameters. pub struct Kzg { - trusted_setup: KZGSettings, + trusted_setup: KzgSettings, } impl Kzg { @@ -32,7 +32,7 @@ impl Kzg { /// The number of G2 points should be equal to 65. pub fn new_from_trusted_setup(trusted_setup: TrustedSetup) -> Result { Ok(Self { - trusted_setup: KZGSettings::load_trusted_setup( + trusted_setup: KzgSettings::load_trusted_setup( trusted_setup.g1_points(), trusted_setup.g2_points(), ) @@ -47,7 +47,7 @@ impl Kzg { #[deprecated] pub fn new_from_file(file_path: PathBuf) -> Result { Ok(Self { - trusted_setup: KZGSettings::load_trusted_setup_file(file_path) + trusted_setup: KzgSettings::load_trusted_setup_file(file_path) .map_err(Error::InvalidTrustedSetup)?, }) } @@ -58,7 +58,7 @@ impl Kzg { blob: Blob, kzg_commitment: KzgCommitment, ) -> Result { - c_kzg::KZGProof::compute_blob_kzg_proof(blob, kzg_commitment.into(), &self.trusted_setup) + c_kzg::KzgProof::compute_blob_kzg_proof(blob, kzg_commitment.into(), &self.trusted_setup) .map_err(Error::KzgProofComputationFailed) .map(|proof| KzgProof(proof.to_bytes().into_inner())) } @@ -70,7 +70,7 @@ impl Kzg { kzg_commitment: KzgCommitment, kzg_proof: KzgProof, ) -> Result { - c_kzg::KZGProof::verify_blob_kzg_proof( + c_kzg::KzgProof::verify_blob_kzg_proof( blob, kzg_commitment.into(), kzg_proof.into(), @@ -100,7 +100,7 @@ impl Kzg { .map(|proof| Bytes48::from_bytes(&proof.0)) .collect::, _>>() .map_err(Error::InvalidBytes)?; - c_kzg::KZGProof::verify_blob_kzg_proof_batch( + c_kzg::KzgProof::verify_blob_kzg_proof_batch( blobs, &commitments_bytes, &proofs_bytes, @@ -111,7 +111,7 @@ impl Kzg { /// Converts a blob to a kzg commitment. pub fn blob_to_kzg_commitment(&self, blob: Blob) -> Result { - c_kzg::KZGCommitment::blob_to_kzg_commitment(blob, &self.trusted_setup) + c_kzg::KzgCommitment::blob_to_kzg_commitment(blob, &self.trusted_setup) .map_err(Error::InvalidBlob) .map(|com| KzgCommitment(com.to_bytes().into_inner())) } diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index f80ae74f305..811d7bb16b1 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -24,8 +24,9 @@ use types::{ ContributionAndProof, Domain, Epoch, EthSpec, Fork, Graffiti, Hash256, Keypair, PublicKeyBytes, SelectionProof, Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecar, SignedBlobSidecarList, SignedContributionAndProof, SignedRoot, SignedValidatorRegistrationData, - Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, SyncCommitteeMessage, - SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, SignedVoluntaryExit, VoluntaryExit, + SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, + SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, + VoluntaryExit, }; use validator_dir::ValidatorDir; From 51a07b09c1b380561b41a52404f4e21f4bc75f8f Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Thu, 16 Mar 2023 12:04:04 +0100 Subject: [PATCH 41/96] add `deposit_receipts` --- beacon_node/execution_layer/src/engine_api.rs | 9 +++- .../src/engine_api/json_structures.rs | 54 ++++++++++++++++++- beacon_node/execution_layer/src/lib.rs | 11 ++++ .../test_utils/execution_block_generator.rs | 1 + consensus/types/src/eip6110.rs | 37 +++++++++++++ consensus/types/src/eth_spec.rs | 6 ++- consensus/types/src/execution_payload.rs | 5 ++ consensus/types/src/lib.rs | 6 ++- 8 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 consensus/types/src/eip6110.rs diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 009183d7ab9..590b1ffe5ec 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -10,7 +10,7 @@ use eth2::types::{SsePayloadAttributes, SsePayloadAttributesV1, SsePayloadAttrib pub use ethers_core::types::Transaction; use ethers_core::utils::rlp::{self, Decodable, Rlp}; use http::deposit_methods::RpcError; -pub use json_structures::{JsonWithdrawal, TransitionConfigurationV1}; +pub use json_structures::{JsonDepositReceipt, JsonWithdrawal, TransitionConfigurationV1}; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use std::convert::TryFrom; @@ -51,6 +51,7 @@ pub enum Error { PayloadConversionLogicFlaw, SszError(ssz_types::Error), DeserializeWithdrawals(ssz_types::Error), + DeserializeDepositReceipts(ssz_types::Error), BuilderApi(builder_client::Error), IncorrectStateVariant, RequiredMethodUnsupported(&'static str), @@ -189,6 +190,8 @@ pub struct ExecutionBlockWithTransactions { pub transactions: Vec, #[superstruct(only(Capella, Eip4844))] pub withdrawals: Vec, + #[superstruct(only(Eip4844))] + pub deposit_receipts: Vec, } impl TryFrom> for ExecutionBlockWithTransactions { @@ -267,6 +270,10 @@ impl TryFrom> for ExecutionBlockWithTransactions .into_iter() .map(|withdrawal| withdrawal.into()) .collect(), + deposit_receipts: Vec::from(block.deposit_receipts) + .into_iter() + .map(|receipt| receipt.into()) + .collect(), }) } }; diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index e84e8dd8175..bd5bd8e41a3 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -1,10 +1,12 @@ use super::*; +use eth2::types::PublicKey; +use eth2::types::Signature; use serde::{Deserialize, Serialize}; use strum::EnumString; use superstruct::superstruct; use types::blobs_sidecar::KzgCommitments; use types::{ - Blobs, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, + Blobs, DepositReceipt, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, VariableList, Withdrawal, }; @@ -101,6 +103,8 @@ pub struct JsonExecutionPayload { pub transactions: Transactions, #[superstruct(only(V2, V3))] pub withdrawals: VariableList, + #[superstruct(only(V3))] + pub deposit_receipts: VariableList, } impl From> for JsonExecutionPayloadV1 { @@ -173,6 +177,12 @@ impl From> for JsonExecutionPayloadV3 .map(Into::into) .collect::>() .into(), + deposit_receipts: payload + .deposit_receipts + .into_iter() + .map(Into::into) + .collect::>() + .into(), } } } @@ -257,6 +267,12 @@ impl From> for ExecutionPayloadEip4844 .map(Into::into) .collect::>() .into(), + deposit_receipts: payload + .deposit_receipts + .into_iter() + .map(Into::into) + .collect::>() + .into(), } } } @@ -352,6 +368,42 @@ impl From for Withdrawal { } } +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct JsonDepositReceipt { + pub pubkey: PublicKey, + pub withdrawal_credentials: Hash256, + #[serde(with = "eth2_serde_utils::u64_hex_be")] + pub amount: u64, + pub signature: Signature, + #[serde(with = "eth2_serde_utils::u64_hex_be")] + pub index: u64, +} + +impl From for JsonDepositReceipt { + fn from(deposit_receipt: DepositReceipt) -> Self { + Self { + pubkey: deposit_receipt.pubkey, + withdrawal_credentials: deposit_receipt.withdrawal_credentials, + amount: deposit_receipt.amount, + signature: deposit_receipt.signature, + index: deposit_receipt.index, + } + } +} + +impl From for DepositReceipt { + fn from(json_deposit_receipt: JsonDepositReceipt) -> Self { + Self { + pubkey: json_deposit_receipt.pubkey, + withdrawal_credentials: json_deposit_receipt.withdrawal_credentials, + amount: json_deposit_receipt.amount, + signature: json_deposit_receipt.signature, + index: json_deposit_receipt.index, + } + } +} + #[superstruct( variants(V1, V2), variant_attributes( diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 2160c06e4dc..06192c9098c 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1784,6 +1784,16 @@ impl ExecutionLayer { .collect(), ) .map_err(ApiError::DeserializeWithdrawals)?; + + let deposit_receipts = VariableList::new( + eip4844_block + .deposit_receipts + .into_iter() + .map(Into::into) + .collect(), + ) + .map_err(ApiError::DeserializeDepositReceipts)?; + ExecutionPayload::Eip4844(ExecutionPayloadEip4844 { parent_hash: eip4844_block.parent_hash, fee_recipient: eip4844_block.fee_recipient, @@ -1801,6 +1811,7 @@ impl ExecutionLayer { block_hash: eip4844_block.block_hash, transactions: convert_transactions(eip4844_block.transactions)?, withdrawals, + deposit_receipts, }) } }; diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index fe3c6f274b8..a6db7eee594 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -554,6 +554,7 @@ impl ExecutionBlockGenerator { block_hash: ExecutionBlockHash::zero(), transactions: vec![].into(), withdrawals: pa.withdrawals.clone().into(), + deposit_receipts: vec![].into(), // TODO: Check if deposit receipts should be part of PayloadAttributesV2 }) } _ => unreachable!(), diff --git a/consensus/types/src/eip6110.rs b/consensus/types/src/eip6110.rs new file mode 100644 index 00000000000..2050f0834a6 --- /dev/null +++ b/consensus/types/src/eip6110.rs @@ -0,0 +1,37 @@ +use crate::test_utils::TestRandom; +use crate::*; +use serde_derive::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + arbitrary::Arbitrary, + Debug, + PartialEq, + // Eq, + Hash, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, +)] +pub struct DepositReceipt { + pub pubkey: PublicKey, + pub withdrawal_credentials: Hash256, + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub amount: u64, + pub signature: Signature, + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub index: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_and_tree_hash_tests!(DepositReceipt); +} diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 7a78dd5800c..66316a699d0 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -108,6 +108,7 @@ pub trait EthSpec: type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq; type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BytesPerFieldElement: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxDepositReceiptsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* * Derived values (set these CAREFULLY) */ @@ -303,6 +304,7 @@ impl EthSpec for MainnetEthSpec { type SlotsPerEth1VotingPeriod = U2048; // 64 epochs * 32 slots per epoch type MaxBlsToExecutionChanges = U16; type MaxWithdrawalsPerPayload = U16; + type MaxDepositReceiptsPerPayload = U8192; fn default_spec() -> ChainSpec { ChainSpec::mainnet() @@ -352,7 +354,8 @@ impl EthSpec for MinimalEthSpec { MaxExtraDataBytes, MaxBlsToExecutionChanges, MaxBlobsPerBlock, - BytesPerFieldElement + BytesPerFieldElement, + MaxDepositReceiptsPerPayload }); fn default_spec() -> ChainSpec { @@ -402,6 +405,7 @@ impl EthSpec for GnosisEthSpec { type FieldElementsPerBlob = U4096; type BytesPerFieldElement = U32; type BytesPerBlob = U131072; + type MaxDepositReceiptsPerPayload = U8192; fn default_spec() -> ChainSpec { ChainSpec::gnosis() diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 7762afe9184..2e02326de4f 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -14,6 +14,9 @@ pub type Transactions = VariableList< pub type Withdrawals = VariableList::MaxWithdrawalsPerPayload>; +pub type DepositReceipts = + VariableList::MaxDepositReceiptsPerPayload>; + #[superstruct( variants(Merge, Capella, Eip4844), variant_attributes( @@ -87,6 +90,8 @@ pub struct ExecutionPayload { pub transactions: Transactions, #[superstruct(only(Capella, Eip4844))] pub withdrawals: Withdrawals, + #[superstruct(only(Eip4844))] + pub deposit_receipts: DepositReceipts, } impl<'a, T: EthSpec> ExecutionPayloadRef<'a, T> { diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 221dda31bc4..b38a6080b6c 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -74,6 +74,7 @@ pub mod voluntary_exit; #[macro_use] pub mod slot_epoch_macros; pub mod config_and_preset; +pub mod eip6110; pub mod execution_block_header; pub mod fork_context; pub mod participation_flags; @@ -131,14 +132,15 @@ pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH}; pub use crate::deposit_data::DepositData; pub use crate::deposit_message::DepositMessage; pub use crate::deposit_tree_snapshot::{DepositTreeSnapshot, FinalizedExecutionBlock}; +pub use crate::eip6110::DepositReceipt; pub use crate::enr_fork_id::EnrForkId; pub use crate::eth1_data::Eth1Data; pub use crate::eth_spec::EthSpecId; pub use crate::execution_block_hash::ExecutionBlockHash; pub use crate::execution_block_header::ExecutionBlockHeader; pub use crate::execution_payload::{ - ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, - ExecutionPayloadRef, Transaction, Transactions, Withdrawals, + DepositReceipts, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, + ExecutionPayloadMerge, ExecutionPayloadRef, Transaction, Transactions, Withdrawals, }; pub use crate::execution_payload_header::{ ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderEip4844, From a0157f46b0e288651cf4907ec21209cb588e6658 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Thu, 16 Mar 2023 16:32:03 +0100 Subject: [PATCH 42/96] modify `ExecutionPayloadHeader`& `BeaconState` --- beacon_node/execution_layer/src/lib.rs | 18 +++++++++--------- beacon_node/store/src/partial_beacon_state.rs | 15 +++++++++++---- .../state_processing/src/upgrade/capella.rs | 2 ++ .../state_processing/src/upgrade/eip4844.rs | 1 + consensus/types/src/beacon_state.rs | 3 +++ .../types/src/execution_payload_header.rs | 5 +++++ 6 files changed, 31 insertions(+), 13 deletions(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 06192c9098c..9702e210569 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -43,13 +43,13 @@ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::consts::eip4844::BLOB_TX_TYPE; use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlobTransaction}; -use types::Withdrawals; use types::{ blobs_sidecar::{Blobs, KzgCommitments}, BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, ForkName, }; use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; +use types::{DepositReceipt, Withdrawals}; use types::{ ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Transaction, Uint256, @@ -1785,14 +1785,14 @@ impl ExecutionLayer { ) .map_err(ApiError::DeserializeWithdrawals)?; - let deposit_receipts = VariableList::new( - eip4844_block - .deposit_receipts - .into_iter() - .map(Into::into) - .collect(), - ) - .map_err(ApiError::DeserializeDepositReceipts)?; + let deposit_receipts = eip4844_block + .deposit_receipts + .into_iter() + .map(Into::into) + .collect::>(); + + let deposit_receipts = VariableList::new(deposit_receipts) + .map_err(ApiError::DeserializeDepositReceipts)?; ExecutionPayload::Eip4844(ExecutionPayloadEip4844 { parent_hash: eip4844_block.parent_hash, diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index 55697bd3160..b642c2752c6 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -114,6 +114,9 @@ where #[ssz(skip_serializing, skip_deserializing)] #[superstruct(only(Capella, Eip4844))] pub historical_summaries: Option>, + + #[superstruct(only(Capella, Eip4844))] + pub deposit_receipts_start_index: u64, } /// Implement the conversion function from BeaconState -> PartialBeaconState. @@ -223,7 +226,8 @@ impl PartialBeaconState { inactivity_scores, latest_execution_payload_header, next_withdrawal_index, - next_withdrawal_validator_index + next_withdrawal_validator_index, + deposit_receipts_start_index ], [historical_summaries] ), @@ -240,7 +244,8 @@ impl PartialBeaconState { inactivity_scores, latest_execution_payload_header, next_withdrawal_index, - next_withdrawal_validator_index + next_withdrawal_validator_index, + deposit_receipts_start_index ], [historical_summaries] ), @@ -468,7 +473,8 @@ impl TryInto> for PartialBeaconState { inactivity_scores, latest_execution_payload_header, next_withdrawal_index, - next_withdrawal_validator_index + next_withdrawal_validator_index, + deposit_receipts_start_index ], [historical_summaries] ), @@ -484,7 +490,8 @@ impl TryInto> for PartialBeaconState { inactivity_scores, latest_execution_payload_header, next_withdrawal_index, - next_withdrawal_validator_index + next_withdrawal_validator_index, + deposit_receipts_start_index ], [historical_summaries] ), diff --git a/consensus/state_processing/src/upgrade/capella.rs b/consensus/state_processing/src/upgrade/capella.rs index 3b933fac37a..babea82cc2e 100644 --- a/consensus/state_processing/src/upgrade/capella.rs +++ b/consensus/state_processing/src/upgrade/capella.rs @@ -66,6 +66,8 @@ pub fn upgrade_to_capella( pubkey_cache: mem::take(&mut pre.pubkey_cache), exit_cache: mem::take(&mut pre.exit_cache), tree_hash_cache: mem::take(&mut pre.tree_hash_cache), + // EIP-6110 + deposit_receipts_start_index: 0, }); *pre_state = post; diff --git a/consensus/state_processing/src/upgrade/eip4844.rs b/consensus/state_processing/src/upgrade/eip4844.rs index 4f6ff9d1943..c6e8c15c185 100644 --- a/consensus/state_processing/src/upgrade/eip4844.rs +++ b/consensus/state_processing/src/upgrade/eip4844.rs @@ -67,6 +67,7 @@ pub fn upgrade_to_eip4844( pubkey_cache: mem::take(&mut pre.pubkey_cache), exit_cache: mem::take(&mut pre.exit_cache), tree_hash_cache: mem::take(&mut pre.tree_hash_cache), + deposit_receipts_start_index: pre.deposit_receipts_start_index, }); *pre_state = post; diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index c98df48d14e..4f3a68e3b4f 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -309,6 +309,9 @@ where // Deep history valid from Capella onwards. #[superstruct(only(Capella, Eip4844))] pub historical_summaries: VariableList, + #[superstruct(only(Capella, Eip4844), partial_getter(copy))] + #[serde(with = "eth2_serde_utils::quoted_u64")] + pub deposit_receipts_start_index: u64, // Caching (not in the spec) #[serde(skip_serializing, skip_deserializing)] diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 4dc79ddc999..fde2fa8880d 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -81,6 +81,9 @@ pub struct ExecutionPayloadHeader { #[superstruct(only(Capella, Eip4844))] #[superstruct(getter(copy))] pub withdrawals_root: Hash256, + #[superstruct(only(Eip4844))] + #[superstruct(getter(copy))] + pub deposit_receipts_root: Hash256, } impl ExecutionPayloadHeader { @@ -155,6 +158,7 @@ impl ExecutionPayloadHeaderCapella { block_hash: self.block_hash, transactions_root: self.transactions_root, withdrawals_root: self.withdrawals_root, + deposit_receipts_root: Hash256::zero(), } } } @@ -220,6 +224,7 @@ impl<'a, T: EthSpec> From<&'a ExecutionPayloadEip4844> for ExecutionPayloadHe block_hash: payload.block_hash, transactions_root: payload.transactions.tree_hash_root(), withdrawals_root: payload.withdrawals.tree_hash_root(), + deposit_receipts_root: payload.deposit_receipts.tree_hash_root(), } } } From 57afc5e73a6723ba8b32aa45f2f172a8517e0518 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Fri, 17 Mar 2023 14:23:58 +0100 Subject: [PATCH 43/96] add `process_deposit_receipt`&`process_operations` --- .../src/per_block_processing.rs | 24 +++++++++++++++++++ .../process_operations.rs | 19 +++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index b2d7e000723..1bb7538bea2 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -416,6 +416,30 @@ pub fn process_execution_payload>( Ok(()) } +pub const UNSET_DEPOSIT_RECEIPTS_START_INDEX: u64 = std::u64::MAX; +pub fn process_deposit_receipt( + state: &mut BeaconState, + deposit_receipts: &DepositReceipt, +) -> Result<(), BlockProcessingError> { + /* + // Set deposit receipt start index + let start_index = state.deposit_receipts_start_index().map_err(|_| BlockProcessingError::DepositReceiptError)?; + if start_index == UNSET_DEPOSIT_RECEIPTS_START_INDEX { + *state.deposit_receipts_start_index_mut().map_err(|_| BlockProcessingError::DepositReceiptError)? = deposit_receipt.index; + } + + // Process the deposit (replace this with the actual deposit processing logic) + let pubkey = &deposit_receipt.pubkey; + let withdrawal_credentials = &deposit_receipt.withdrawal_credentials; + let amount = deposit_receipt.amount; + let signature = &deposit_receipt.signature; + */ + + // Add the logic to process the deposit here + + Ok(()) +} + /// These functions will definitely be called before the merge. Their entire purpose is to check if /// the merge has happened or if we're on the transition block. Thus we don't want to propagate /// errors from the `BeaconState` being an earlier variant than `BeaconStateMerge` as we'd have to diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 8a6163f29b9..4ba4cb6beee 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -16,6 +16,12 @@ pub fn process_operations>( ctxt: &mut ConsensusContext, spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { + let unprocessed_deposits_count = state + .eth1_data() + .deposit_count + .saturating_sub(state.eth1_deposit_index()); + let max_deposits = ::MaxDeposits::to_u64(); + process_proposer_slashings( state, block_body.proposer_slashings(), @@ -31,12 +37,21 @@ pub fn process_operations>( spec, )?; process_attestations(state, block_body, verify_signatures, ctxt, spec)?; + assert_eq!( + block_body.deposits().len() as usize, + std::cmp::min(max_deposits as usize, unprocessed_deposits_count as usize), + "Number of deposits in block does not match the minimum of the maximum number of deposits and the number of unprocessed deposits" + ); process_deposits(state, block_body.deposits(), spec)?; process_exits(state, block_body.voluntary_exits(), verify_signatures, spec)?; - if let Ok(bls_to_execution_changes) = block_body.bls_to_execution_changes() { - process_bls_to_execution_changes(state, bls_to_execution_changes, verify_signatures, spec)?; + /* + if let Ok(payload) = block_body.execution_payload() { + if is_execution_enabled(state, block_body) { + process_deposit_receipt(state, payload.deposit_receipts())?; + } } + */ Ok(()) } From b20fa959feb025996e6df90bfbd8716675a280a0 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Sun, 19 Mar 2023 15:11:41 +0100 Subject: [PATCH 44/96] update `process_deposit_receipt` --- .../src/per_block_processing.rs | 27 ++++++++++--------- .../src/per_block_processing/errors.rs | 1 + .../process_operations.rs | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 1bb7538bea2..080a7f467a2 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -419,23 +419,26 @@ pub fn process_execution_payload>( pub const UNSET_DEPOSIT_RECEIPTS_START_INDEX: u64 = std::u64::MAX; pub fn process_deposit_receipt( state: &mut BeaconState, - deposit_receipts: &DepositReceipt, + deposit_receipt: &DepositReceipt, ) -> Result<(), BlockProcessingError> { - /* // Set deposit receipt start index - let start_index = state.deposit_receipts_start_index().map_err(|_| BlockProcessingError::DepositReceiptError)?; + let start_index = state + .deposit_receipts_start_index() + .map_err(|_| BlockProcessingError::DepositReceiptError)?; if start_index == UNSET_DEPOSIT_RECEIPTS_START_INDEX { - *state.deposit_receipts_start_index_mut().map_err(|_| BlockProcessingError::DepositReceiptError)? = deposit_receipt.index; + *state + .deposit_receipts_start_index_mut() + .map_err(|_| BlockProcessingError::DepositReceiptError)? = deposit_receipt.index; } - - // Process the deposit (replace this with the actual deposit processing logic) - let pubkey = &deposit_receipt.pubkey; - let withdrawal_credentials = &deposit_receipt.withdrawal_credentials; - let amount = deposit_receipt.amount; - let signature = &deposit_receipt.signature; - */ - // Add the logic to process the deposit here + /* TODO: Check how apply_deposit works: Create a DepositData instance (?) from the DepositReceipt fields + let deposit_data = DepositData { + pubkey: deposit_receipt.pubkey.into(), + withdrawal_credentials: deposit_receipt.withdrawal_credentials, + amount: deposit_receipt.amount, + signature: deposit_receipt.signature.into(), + }; + */ Ok(()) } diff --git a/consensus/state_processing/src/per_block_processing/errors.rs b/consensus/state_processing/src/per_block_processing/errors.rs index 5c34afd593e..0370945a16a 100644 --- a/consensus/state_processing/src/per_block_processing/errors.rs +++ b/consensus/state_processing/src/per_block_processing/errors.rs @@ -95,6 +95,7 @@ pub enum BlockProcessingError { length: usize, }, WithdrawalCredentialsInvalid, + DepositReceiptError, } impl From for BlockProcessingError { diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 4ba4cb6beee..c64620b1b9a 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -45,7 +45,7 @@ pub fn process_operations>( process_deposits(state, block_body.deposits(), spec)?; process_exits(state, block_body.voluntary_exits(), verify_signatures, spec)?; - /* + /* TODO: Add deposit_receipts to AbstractExecPayload if let Ok(payload) = block_body.execution_payload() { if is_execution_enabled(state, block_body) { process_deposit_receipt(state, payload.deposit_receipts())?; From 541ff94da40f6651b252ffe6506627565eb99093 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Mon, 20 Mar 2023 10:45:51 +0100 Subject: [PATCH 45/96] finish `process_deposit_receipt` --- .../state_processing/src/per_block_processing.rs | 16 ++++++++++++---- .../per_block_processing/process_operations.rs | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 080a7f467a2..adea2df1c59 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -7,6 +7,7 @@ use std::borrow::Cow; use tree_hash::TreeHash; use types::*; +use self::process_operations::process_deposit; pub use self::verify_attester_slashing::{ get_slashable_indices, get_slashable_indices_modular, verify_attester_slashing, }; @@ -420,6 +421,7 @@ pub const UNSET_DEPOSIT_RECEIPTS_START_INDEX: u64 = std::u64::MAX; pub fn process_deposit_receipt( state: &mut BeaconState, deposit_receipt: &DepositReceipt, + spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { // Set deposit receipt start index let start_index = state @@ -431,14 +433,20 @@ pub fn process_deposit_receipt( .map_err(|_| BlockProcessingError::DepositReceiptError)? = deposit_receipt.index; } - /* TODO: Check how apply_deposit works: Create a DepositData instance (?) from the DepositReceipt fields + // Create a Deposit object from the DepositReceipt let deposit_data = DepositData { - pubkey: deposit_receipt.pubkey.into(), + pubkey: deposit_receipt.pubkey.clone().into(), withdrawal_credentials: deposit_receipt.withdrawal_credentials, amount: deposit_receipt.amount, - signature: deposit_receipt.signature.into(), + signature: deposit_receipt.signature.clone().into(), + }; + let deposit = Deposit { + proof: FixedVector::from_elem(Hash256::zero()), // Set an empty proof, since it's not included in DepositReceipt + data: deposit_data, }; - */ + + // Call process_deposit with the created Deposit object + process_deposit(state, &deposit, spec, true)?; Ok(()) } diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index c64620b1b9a..526986a2f9a 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -38,7 +38,7 @@ pub fn process_operations>( )?; process_attestations(state, block_body, verify_signatures, ctxt, spec)?; assert_eq!( - block_body.deposits().len() as usize, + block_body.deposits().len(), std::cmp::min(max_deposits as usize, unprocessed_deposits_count as usize), "Number of deposits in block does not match the minimum of the maximum number of deposits and the number of unprocessed deposits" ); From 37adbf00f0a41684a2747b86a51b771424ad37c5 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Mon, 20 Mar 2023 14:13:59 +0100 Subject: [PATCH 46/96] mod. `process_operations` and `payload.rs` --- .../process_operations.rs | 7 ++-- consensus/types/src/payload.rs | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 526986a2f9a..903ee56ec9d 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -45,13 +45,14 @@ pub fn process_operations>( process_deposits(state, block_body.deposits(), spec)?; process_exits(state, block_body.voluntary_exits(), verify_signatures, spec)?; - /* TODO: Add deposit_receipts to AbstractExecPayload if let Ok(payload) = block_body.execution_payload() { if is_execution_enabled(state, block_body) { - process_deposit_receipt(state, payload.deposit_receipts())?; + let deposit_receipts = payload.deposit_receipts()?; + for deposit_receipt in deposit_receipts.iter() { + process_deposit_receipt(state, deposit_receipt, spec)?; + } } } - */ Ok(()) } diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 9ec2d9f30a2..22aacc0e331 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -39,6 +39,7 @@ pub trait ExecPayload: Debug + Clone + PartialEq + Hash + TreeHash + fn transactions(&self) -> Option<&Transactions>; /// fork-specific fields fn withdrawals_root(&self) -> Result; + fn deposit_receipts(&self) -> Result, Error>; /// Is this a default payload with 0x0 roots for transactions and withdrawals? fn is_default_with_zero_roots(&self) -> bool; @@ -267,6 +268,20 @@ impl ExecPayload for FullPayload { // For full payloads the empty/zero distinction does not exist. self.is_default_with_zero_roots() } + + fn deposit_receipts(&self) -> Result, Error> { + match self { + FullPayload::Merge(_) => { + // Return an error for the Merge variant + Err(Error::IncorrectStateVariant) + } + FullPayload::Capella(_) => { + // Return an error for the Capella variant + Err(Error::IncorrectStateVariant) + } + FullPayload::Eip4844(ref inner) => Ok(inner.execution_payload.deposit_receipts.clone()), + } + } } impl FullPayload { @@ -376,6 +391,20 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { // For full payloads the empty/zero distinction does not exist. self.is_default_with_zero_roots() } + + fn deposit_receipts(&self) -> Result, Error> { + match self { + FullPayloadRef::Merge(_) => { + // Return an error for the Merge variant + Err(Error::IncorrectStateVariant) + } + FullPayloadRef::Capella(_) => { + // Return an error for the Capella variant + Err(Error::IncorrectStateVariant) + } + FullPayloadRef::Eip4844(inner) => Ok(inner.execution_payload.deposit_receipts.clone()), + } + } } impl AbstractExecPayload for FullPayload { @@ -548,6 +577,10 @@ impl ExecPayload for BlindedPayload { fn is_default_with_empty_roots(&self) -> bool { self.to_ref().is_default_with_empty_roots() } + + fn deposit_receipts(&self) -> Result, Error> { + todo!() + } } impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { @@ -640,6 +673,10 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { payload.is_default_with_empty_roots() }) } + + fn deposit_receipts(&self) -> Result, Error> { + todo!() + } } macro_rules! impl_exec_payload_common { @@ -710,6 +747,10 @@ macro_rules! impl_exec_payload_common { let g = $g; g(self) } + + fn deposit_receipts(&self) -> Result, Error> { + todo!() + } } impl From<$wrapped_type> for $wrapper_type { From 81b0db6d1395167c8ca84fd351c734920ca8f20b Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Mon, 20 Mar 2023 17:30:18 +0100 Subject: [PATCH 47/96] modify `initialize_beacon_state_from_eth1` --- consensus/state_processing/src/genesis.rs | 102 ++++++---------------- consensus/types/src/chain_spec.rs | 18 ++++ 2 files changed, 47 insertions(+), 73 deletions(-) diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 3f9328f4d5c..275b214c8f7 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -2,15 +2,12 @@ use super::per_block_processing::{ errors::BlockProcessingError, process_operations::process_deposit, }; use crate::common::DepositDataTree; -use crate::upgrade::{ - upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_eip4844, -}; +use crate::per_block_processing::UNSET_DEPOSIT_RECEIPTS_START_INDEX; use safe_arith::{ArithError, SafeArith}; use tree_hash::TreeHash; use types::DEPOSIT_TREE_DEPTH; use types::*; -/// Initialize a `BeaconState` from genesis data. pub fn initialize_beacon_state_from_eth1( eth1_block_hash: Hash256, eth1_timestamp: u64, @@ -20,7 +17,6 @@ pub fn initialize_beacon_state_from_eth1( ) -> Result, BlockProcessingError> { let genesis_time = eth2_genesis_time(eth1_timestamp, spec)?; let eth1_data = Eth1Data { - // Temporary deposit root deposit_root: Hash256::zero(), deposit_count: deposits.len() as u64, block_hash: eth1_block_hash, @@ -42,80 +38,40 @@ pub fn initialize_beacon_state_from_eth1( process_activations(&mut state, spec)?; - // To support testnets with Altair enabled from genesis, perform a possible state upgrade here. - // This must happen *after* deposits and activations are processed or the calculation of sync - // committees during the upgrade will fail. It's a bit cheeky to do this instead of having - // separate Altair genesis initialization logic, but it turns out that our - // use of `BeaconBlock::empty` in `BeaconState::new` is sufficient to correctly initialise - // the `latest_block_header` as per: - // https://github.com/ethereum/eth2.0-specs/pull/2323 - if spec - .altair_fork_epoch - .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) - { - upgrade_to_altair(&mut state, spec)?; - - state.fork_mut().previous_version = spec.altair_fork_version; - } - - // Similarly, perform an upgrade to the merge if configured from genesis. - if spec - .bellatrix_fork_epoch - .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) - { - // this will set state.latest_execution_payload_header = ExecutionPayloadHeaderMerge::default() - upgrade_to_bellatrix(&mut state, spec)?; - - // Remove intermediate Altair fork from `state.fork`. - state.fork_mut().previous_version = spec.bellatrix_fork_version; - - // Override latest execution payload header. - // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/bellatrix/beacon-chain.md#testing - if let Some(ExecutionPayloadHeader::Merge(ref header)) = execution_payload_header { - *state.latest_execution_payload_header_merge_mut()? = header.clone(); - } - } - - // Upgrade to capella if configured from genesis - if spec - .capella_fork_epoch - .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) - { - upgrade_to_capella(&mut state, spec)?; - - // Remove intermediate Bellatrix fork from `state.fork`. - state.fork_mut().previous_version = spec.capella_fork_version; - - // Override latest execution payload header. - // See https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#testing - if let Some(ExecutionPayloadHeader::Capella(ref header)) = execution_payload_header { - *state.latest_execution_payload_header_capella_mut()? = header.clone(); - } - } - - // Upgrade to eip4844 if configured from genesis - if spec - .eip4844_fork_epoch - .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) - { - upgrade_to_eip4844(&mut state, spec)?; - - // Remove intermediate Capella fork from `state.fork`. - state.fork_mut().previous_version = spec.eip4844_fork_version; - - // Override latest execution payload header. - // See https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#testing - if let Some(ExecutionPayloadHeader::Eip4844(header)) = execution_payload_header { - *state.latest_execution_payload_header_eip4844_mut()? = header; - } - } - // Now that we have our validators, initialize the caches (including the committees) state.build_all_caches(spec)?; // Set genesis validators root for domain separation and chain versioning *state.genesis_validators_root_mut() = state.update_validators_tree_hash_cache()?; + // Set fork version to EIP6110 + state.fork_mut().previous_version = spec.eip6110_fork_version; + state.fork_mut().current_version = spec.eip6110_fork_version; + + // Add deposit_receipts_start_index field with the value UNSET_DEPOSIT_RECEIPTS_START_INDEX + *state.deposit_receipts_start_index_mut()? = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + + // Initialize the execution payload header + if let Some(header) = execution_payload_header { + match state.latest_execution_payload_header_mut()? { + ExecutionPayloadHeaderRefMut::Merge(header_mut) => { + if let ExecutionPayloadHeader::Merge(header) = header { + *header_mut = header; + } + } + ExecutionPayloadHeaderRefMut::Capella(header_mut) => { + if let ExecutionPayloadHeader::Capella(header) = header { + *header_mut = header; + } + } + ExecutionPayloadHeaderRefMut::Eip4844(header_mut) => { + if let ExecutionPayloadHeader::Eip4844(header) = header { + *header_mut = header; + } + } // TODO: Should a Eip6110 or upgrade_to_eip6110() be added? + } + } + Ok(state) } diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 61d46b6cce5..806632a6021 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -167,6 +167,12 @@ pub struct ChainSpec { pub eip4844_fork_version: [u8; 4], pub eip4844_fork_epoch: Option, + /* + * Eip6110 hard fork params + */ + pub eip6110_fork_version: [u8; 4], + pub eip6110_fork_epoch: Option, + /* * Networking */ @@ -642,6 +648,12 @@ impl ChainSpec { eip4844_fork_version: [0x04, 0x00, 0x00, 0x00], eip4844_fork_epoch: None, + /* + * Eip6110 hard fork params + */ + eip6110_fork_version: [0x05, 0x00, 0x00, 0x00], + eip6110_fork_epoch: None, + /* * Network specific */ @@ -879,6 +891,12 @@ impl ChainSpec { eip4844_fork_version: [0x04, 0x00, 0x00, 0x64], eip4844_fork_epoch: None, + /* + * Eip6110 hard fork params + */ + eip6110_fork_version: [0x05, 0x00, 0x00, 0x64], + eip6110_fork_epoch: None, + /* * Network specific */ From ef699e3545a4f342297c1643af755c5914a7bf95 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Thu, 23 Mar 2023 16:07:05 +0100 Subject: [PATCH 48/96] fix `consensus-spec-tests` --- .../src/engine_api/json_structures.rs | 16 ++++---- consensus/types/src/eip6110.rs | 4 +- consensus/types/src/payload.rs | 37 +++++++++++++------ 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index bd5bd8e41a3..0384aad4886 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -1,6 +1,6 @@ use super::*; -use eth2::types::PublicKey; -use eth2::types::Signature; +use eth2::types::PublicKeyBytes; +use eth2::types::SignatureBytes; use serde::{Deserialize, Serialize}; use strum::EnumString; use superstruct::superstruct; @@ -371,11 +371,11 @@ impl From for Withdrawal { #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct JsonDepositReceipt { - pub pubkey: PublicKey, + pub pubkey: PublicKeyBytes, pub withdrawal_credentials: Hash256, #[serde(with = "eth2_serde_utils::u64_hex_be")] pub amount: u64, - pub signature: Signature, + pub signature: SignatureBytes, #[serde(with = "eth2_serde_utils::u64_hex_be")] pub index: u64, } @@ -383,10 +383,10 @@ pub struct JsonDepositReceipt { impl From for JsonDepositReceipt { fn from(deposit_receipt: DepositReceipt) -> Self { Self { - pubkey: deposit_receipt.pubkey, + pubkey: deposit_receipt.pubkey.decompress().unwrap().into(), // Convert PublicKeyBytes to PublicKey withdrawal_credentials: deposit_receipt.withdrawal_credentials, amount: deposit_receipt.amount, - signature: deposit_receipt.signature, + signature: (deposit_receipt.signature).try_into().unwrap(), // Convert SignatureBytes to Signature index: deposit_receipt.index, } } @@ -395,10 +395,10 @@ impl From for JsonDepositReceipt { impl From for DepositReceipt { fn from(json_deposit_receipt: JsonDepositReceipt) -> Self { Self { - pubkey: json_deposit_receipt.pubkey, + pubkey: json_deposit_receipt.pubkey.into(), withdrawal_credentials: json_deposit_receipt.withdrawal_credentials, amount: json_deposit_receipt.amount, - signature: json_deposit_receipt.signature, + signature: json_deposit_receipt.signature.into(), index: json_deposit_receipt.index, } } diff --git a/consensus/types/src/eip6110.rs b/consensus/types/src/eip6110.rs index 2050f0834a6..9b375b2aa08 100644 --- a/consensus/types/src/eip6110.rs +++ b/consensus/types/src/eip6110.rs @@ -20,11 +20,11 @@ use tree_hash_derive::TreeHash; TestRandom, )] pub struct DepositReceipt { - pub pubkey: PublicKey, + pub pubkey: PublicKeyBytes, pub withdrawal_credentials: Hash256, #[serde(with = "eth2_serde_utils::quoted_u64")] pub amount: u64, - pub signature: Signature, + pub signature: SignatureBytes, #[serde(with = "eth2_serde_utils::quoted_u64")] pub index: u64, } diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 22aacc0e331..4eee05c1754 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -272,12 +272,12 @@ impl ExecPayload for FullPayload { fn deposit_receipts(&self) -> Result, Error> { match self { FullPayload::Merge(_) => { - // Return an error for the Merge variant - Err(Error::IncorrectStateVariant) + // Return an "empty" or "default" value for the Merge variant + Ok(Vec::new().into()) } FullPayload::Capella(_) => { - // Return an error for the Capella variant - Err(Error::IncorrectStateVariant) + // Return an "empty" or "default" value for the Capella variant + Ok(Vec::new().into()) } FullPayload::Eip4844(ref inner) => Ok(inner.execution_payload.deposit_receipts.clone()), } @@ -395,16 +395,16 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { fn deposit_receipts(&self) -> Result, Error> { match self { FullPayloadRef::Merge(_) => { - // Return an error for the Merge variant - Err(Error::IncorrectStateVariant) + // Return an "empty" or "default" value for the Merge variant + Ok(Vec::new().into()) } FullPayloadRef::Capella(_) => { - // Return an error for the Capella variant - Err(Error::IncorrectStateVariant) + // Return an "empty" or "default" value for the Capella variant + Ok(Vec::new().into()) } - FullPayloadRef::Eip4844(inner) => Ok(inner.execution_payload.deposit_receipts.clone()), + FullPayloadRef::Eip4844(ref inner) => Ok(inner.execution_payload.deposit_receipts.clone()), } - } + } } impl AbstractExecPayload for FullPayload { @@ -675,8 +675,21 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { } fn deposit_receipts(&self) -> Result, Error> { - todo!() - } + match self { + BlindedPayloadRef::Merge(_) => { + // Return an "empty" or "default" value for the Merge variant + Ok(Vec::new().into()) + } + BlindedPayloadRef::Capella(_) => { + // Return an "empty" or "default" value for the Capella variant + Ok(Vec::new().into()) + } + BlindedPayloadRef::Eip4844(_) => { + // Return an "empty" or "default" value for the Eip4844 variant + Ok(Vec::new().into()) + } + } + } } macro_rules! impl_exec_payload_common { From cf6aefb043fd11a7109b49989f820fb1519daf61 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Thu, 23 Mar 2023 16:18:48 +0100 Subject: [PATCH 49/96] cargo fmt --- consensus/types/src/payload.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 4eee05c1754..bbc6423c057 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -402,9 +402,11 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { // Return an "empty" or "default" value for the Capella variant Ok(Vec::new().into()) } - FullPayloadRef::Eip4844(ref inner) => Ok(inner.execution_payload.deposit_receipts.clone()), + FullPayloadRef::Eip4844(ref inner) => { + Ok(inner.execution_payload.deposit_receipts.clone()) + } } - } + } } impl AbstractExecPayload for FullPayload { @@ -689,7 +691,7 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { Ok(Vec::new().into()) } } - } + } } macro_rules! impl_exec_payload_common { From 824d40d80835a2bbdf61f128a08ff4ff35eedb01 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Tue, 28 Mar 2023 17:07:21 +0200 Subject: [PATCH 50/96] Decouple `Eip6110` and `Eip4844` --- Makefile | 2 +- beacon_node/beacon_chain/src/beacon_chain.rs | 39 +++++++- .../beacon_chain/src/blob_verification.rs | 6 +- .../beacon_chain/src/execution_payload.rs | 2 +- beacon_node/beacon_chain/src/test_utils.rs | 5 + beacon_node/execution_layer/src/engine_api.rs | 52 ++++++++-- .../execution_layer/src/engine_api/http.rs | 47 ++++++++- .../src/engine_api/json_structures.rs | 84 ++++++++++++++-- beacon_node/execution_layer/src/lib.rs | 61 +++++++++--- .../test_utils/execution_block_generator.rs | 43 +++++++-- .../src/test_utils/handle_rpc.rs | 95 ++++++++++++++++++- .../src/test_utils/mock_builder.rs | 4 +- .../src/test_utils/mock_execution_layer.rs | 3 + .../execution_layer/src/test_utils/mod.rs | 7 ++ beacon_node/lighthouse_network/src/config.rs | 6 +- .../lighthouse_network/src/rpc/codec/base.rs | 3 + .../src/rpc/codec/ssz_snappy.rs | 20 +++- .../lighthouse_network/src/rpc/protocol.rs | 12 +++ .../lighthouse_network/src/types/pubsub.rs | 8 +- .../lighthouse_network/src/types/topics.rs | 1 + .../lighthouse_network/tests/common.rs | 3 + .../store/src/impls/execution_payload.rs | 3 +- beacon_node/store/src/partial_beacon_state.rs | 64 ++++++++++--- common/eth2/src/types.rs | 8 +- .../eip4844/config.yaml | 4 + .../gnosis/config.yaml | 3 + .../mainnet/config.yaml | 3 + .../sepolia/config.yaml | 4 + consensus/fork_choice/src/fork_choice.rs | 1 + .../src/common/slash_validator.rs | 3 + consensus/state_processing/src/genesis.rs | 7 +- .../src/per_block_processing.rs | 9 +- .../src/per_block_processing/eip6110.rs | 2 + .../per_block_processing/eip6110/eip6110.rs | 1 + .../process_operations.rs | 3 +- .../src/per_epoch_processing.rs | 2 + .../src/per_epoch_processing/eip4844.rs | 75 +++++++++++++++ .../src/per_slot_processing.rs | 5 + consensus/state_processing/src/upgrade.rs | 2 + .../state_processing/src/upgrade/capella.rs | 2 - .../state_processing/src/upgrade/eip4844.rs | 1 - .../state_processing/src/upgrade/eip6110.rs | 72 ++++++++++++++ consensus/types/src/beacon_block.rs | 67 ++++++++++++- consensus/types/src/beacon_block_body.rs | 94 +++++++++++++++++- consensus/types/src/beacon_state.rs | 38 ++++++-- consensus/types/src/chain_spec.rs | 50 ++++++++-- consensus/types/src/eip6110.rs | 14 ++- consensus/types/src/eth_spec.rs | 5 + consensus/types/src/execution_payload.rs | 23 ++++- .../types/src/execution_payload_header.rs | 76 ++++++++++++++- consensus/types/src/fork_context.rs | 7 ++ consensus/types/src/fork_name.rs | 25 ++++- consensus/types/src/lib.rs | 21 ++-- consensus/types/src/payload.rs | 63 +++++++++++- consensus/types/src/signed_beacon_block.rs | 67 ++++++++++++- lcli/src/create_payload_header.rs | 10 +- lcli/src/main.rs | 11 ++- lcli/src/new_testnet.rs | 12 ++- lcli/src/parse_ssz.rs | 3 + scripts/local_testnet/setup.sh | 2 + scripts/local_testnet/vars.env | 1 + scripts/tests/vars.env | 1 + testing/ef_tests/src/cases/common.rs | 1 + .../ef_tests/src/cases/epoch_processing.rs | 22 +++-- testing/ef_tests/src/cases/fork.rs | 2 + testing/ef_tests/src/cases/operations.rs | 3 +- testing/ef_tests/src/cases/transition.rs | 7 ++ testing/ef_tests/src/handler.rs | 4 + testing/ef_tests/src/type_name.rs | 3 + testing/ef_tests/tests/tests.rs | 12 +++ .../src/signing_method/web3signer.rs | 6 ++ 71 files changed, 1293 insertions(+), 134 deletions(-) create mode 100644 consensus/state_processing/src/per_block_processing/eip6110.rs create mode 100644 consensus/state_processing/src/per_block_processing/eip6110/eip6110.rs create mode 100644 consensus/state_processing/src/per_epoch_processing/eip4844.rs create mode 100644 consensus/state_processing/src/upgrade/eip6110.rs diff --git a/Makefile b/Makefile index bf2ad679453..711b783aa69 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ PROFILE ?= release # List of all hard forks. This list is used to set env variables for several tests so that # they run for different forks. -FORKS=phase0 altair merge capella eip4844 +FORKS=phase0 altair merge capella eip4844 eip6110 # Extra flags for Cargo CARGO_INSTALL_EXTRA_FLAGS?= diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index c3a1ed55ecd..0dc7e0dd721 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4439,7 +4439,10 @@ impl BeaconChain { // allows it to run concurrently with things like attestation packing. let prepare_payload_handle = match &state { BeaconState::Base(_) | BeaconState::Altair(_) => None, - BeaconState::Merge(_) | BeaconState::Capella(_) | BeaconState::Eip4844(_) => { + BeaconState::Merge(_) + | BeaconState::Capella(_) + | BeaconState::Eip4844(_) + | BeaconState::Eip6110(_) => { let prepare_payload_handle = get_execution_payload(self.clone(), &state, proposer_index, builder_params)?; Some(prepare_payload_handle) @@ -4779,6 +4782,38 @@ impl BeaconChain { blobs, ) } + BeaconState::Eip6110(_) => { + let (payload, kzg_commitments, blobs) = block_contents + .ok_or(BlockProductionError::MissingExecutionPayload)? + .deconstruct(); + ( + BeaconBlock::Eip6110(BeaconBlockEip6110 { + slot, + proposer_index, + parent_root, + state_root: Hash256::zero(), + body: BeaconBlockBodyEip6110 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings: proposer_slashings.into(), + attester_slashings: attester_slashings.into(), + attestations: attestations.into(), + deposits: deposits.into(), + voluntary_exits: voluntary_exits.into(), + sync_aggregate: sync_aggregate + .ok_or(BlockProductionError::MissingSyncAggregate)?, + execution_payload: payload + .try_into() + .map_err(|_| BlockProductionError::InvalidPayloadFork)?, + bls_to_execution_changes: bls_to_execution_changes.into(), + blob_kzg_commitments: kzg_commitments + .ok_or(BlockProductionError::InvalidPayloadFork)?, + }, + }), + blobs, + ) + } }; let block = SignedBeaconBlock::from_block( @@ -5081,7 +5116,7 @@ impl BeaconChain { } else { let withdrawals = match self.spec.fork_name_at_slot::(prepare_slot) { ForkName::Base | ForkName::Altair | ForkName::Merge => None, - ForkName::Capella | ForkName::Eip4844 => { + ForkName::Capella | ForkName::Eip4844 | ForkName::Eip6110 => { let chain = self.clone(); self.spawn_blocking_handle( move || { diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 1d7f9cc3c52..673dfd42780 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -285,7 +285,7 @@ impl AvailableBlock { | SignedBeaconBlock::Merge(_) => { Ok(AvailableBlock(AvailableBlockInner::Block(beacon_block))) } - SignedBeaconBlock::Eip4844(_) => { + SignedBeaconBlock::Eip4844(_) | SignedBeaconBlock::Eip6110(_) => { match da_check_required { DataAvailabilityCheckRequired::Yes => { // Attempt to reconstruct empty blobs here. @@ -320,7 +320,7 @@ impl AvailableBlock { | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Capella(_) | SignedBeaconBlock::Merge(_) => Err(BlobError::InconsistentFork), - SignedBeaconBlock::Eip4844(_) => { + SignedBeaconBlock::Eip4844(_) | SignedBeaconBlock::Eip6110(_) => { match da_check_required { DataAvailabilityCheckRequired::Yes => Ok(AvailableBlock( AvailableBlockInner::BlockAndBlob(SignedBeaconBlockAndBlobsSidecar { @@ -330,7 +330,7 @@ impl AvailableBlock { )), DataAvailabilityCheckRequired::No => { // Blobs were not verified so we drop them, we'll instead just pass around - // an available `Eip4844` block without blobs. + // an available `Eip4844` or `Eip6110` block without blobs. Ok(AvailableBlock(AvailableBlockInner::Block(beacon_block))) } } diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index 41baa956ab9..78a0432b11a 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -419,7 +419,7 @@ pub fn get_execution_payload< let latest_execution_payload_header_block_hash = state.latest_execution_payload_header()?.block_hash(); let withdrawals = match state { - &BeaconState::Capella(_) | &BeaconState::Eip4844(_) => { + &BeaconState::Capella(_) | &BeaconState::Eip4844(_) | &BeaconState::Eip6110(_) => { Some(get_expected_withdrawals(state, spec)?.into()) } &BeaconState::Merge(_) => None, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index a784c674113..949310bdd7a 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -453,6 +453,7 @@ where shanghai_time, eip4844_time, None, + None, Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec, None, @@ -478,11 +479,15 @@ where let eip4844_time = spec.eip4844_fork_epoch.map(|epoch| { HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); + let eip6110_time = spec.eip6110_fork_epoch.map(|epoch| { + HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() + }); let mock_el = MockExecutionLayer::new( self.runtime.task_executor.clone(), DEFAULT_TERMINAL_BLOCK, shanghai_time, eip4844_time, + eip6110_time, builder_threshold, Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec.clone(), diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 590b1ffe5ec..0769a86a382 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -21,7 +21,10 @@ pub use types::{ ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, Withdrawal, Withdrawals, }; -use types::{ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge}; +use types::{ + DepositReceipt, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadEip6110, + ExecutionPayloadMerge, +}; pub mod auth; pub mod http; @@ -151,7 +154,7 @@ pub struct ExecutionBlock { /// Representation of an execution block with enough detail to reconstruct a payload. #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Eip4844, Eip6110), variant_attributes( derive(Clone, Debug, PartialEq, Serialize, Deserialize,), serde(bound = "T: EthSpec", rename_all = "camelCase"), @@ -182,15 +185,15 @@ pub struct ExecutionBlockWithTransactions { #[serde(with = "ssz_types::serde_utils::hex_var_list")] pub extra_data: VariableList, pub base_fee_per_gas: Uint256, - #[superstruct(only(Eip4844))] + #[superstruct(only(Eip4844, Eip6110))] #[serde(with = "eth2_serde_utils::u256_hex_be")] pub excess_data_gas: Uint256, #[serde(rename = "hash")] pub block_hash: ExecutionBlockHash, pub transactions: Vec, - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Eip4844, Eip6110))] pub withdrawals: Vec, - #[superstruct(only(Eip4844))] + #[superstruct(only(Eip6110))] pub deposit_receipts: Vec, } @@ -270,6 +273,33 @@ impl TryFrom> for ExecutionBlockWithTransactions .into_iter() .map(|withdrawal| withdrawal.into()) .collect(), + }) + } + ExecutionPayload::Eip6110(block) => { + Self::Eip6110(ExecutionBlockWithTransactionsEip6110 { + parent_hash: block.parent_hash, + fee_recipient: block.fee_recipient, + state_root: block.state_root, + receipts_root: block.receipts_root, + logs_bloom: block.logs_bloom, + prev_randao: block.prev_randao, + block_number: block.block_number, + gas_limit: block.gas_limit, + gas_used: block.gas_used, + timestamp: block.timestamp, + extra_data: block.extra_data, + base_fee_per_gas: block.base_fee_per_gas, + excess_data_gas: block.excess_data_gas, + block_hash: block.block_hash, + transactions: block + .transactions + .iter() + .map(|tx| Transaction::decode(&Rlp::new(tx))) + .collect::, _>>()?, + withdrawals: Vec::from(block.withdrawals) + .into_iter() + .map(|withdrawal| withdrawal.into()) + .collect(), deposit_receipts: Vec::from(block.deposit_receipts) .into_iter() .map(|receipt| receipt.into()) @@ -297,6 +327,8 @@ pub struct PayloadAttributes { pub suggested_fee_recipient: Address, #[superstruct(only(V2))] pub withdrawals: Vec, + #[superstruct(only(V2))] + pub deposit_receipts: Vec, } impl PayloadAttributes { @@ -312,6 +344,7 @@ impl PayloadAttributes { prev_randao, suggested_fee_recipient, withdrawals, + deposit_receipts: Vec::new(), }), None => PayloadAttributes::V1(PayloadAttributesV1 { timestamp, @@ -339,6 +372,7 @@ impl From for SsePayloadAttributes { prev_randao, suggested_fee_recipient, withdrawals, + deposit_receipts: _, }) => Self::V2(SsePayloadAttributesV2 { timestamp, prev_randao, @@ -370,7 +404,7 @@ pub struct ProposeBlindedBlockResponse { } #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Eip4844, Eip6110), variant_attributes(derive(Clone, Debug, PartialEq),), map_into(ExecutionPayload), map_ref_into(ExecutionPayloadRef), @@ -385,6 +419,8 @@ pub struct GetPayloadResponse { pub execution_payload: ExecutionPayloadCapella, #[superstruct(only(Eip4844), partial_getter(rename = "execution_payload_eip4844"))] pub execution_payload: ExecutionPayloadEip4844, + #[superstruct(only(Eip6110), partial_getter(rename = "execution_payload_eip6110"))] + pub execution_payload: ExecutionPayloadEip6110, pub block_value: Uint256, } @@ -419,6 +455,10 @@ impl From> for (ExecutionPayload, Uint256) ExecutionPayload::Eip4844(inner.execution_payload), inner.block_value, ), + GetPayloadResponse::Eip6110(inner) => ( + ExecutionPayload::Eip6110(inner.execution_payload), + inner.block_value, + ), } } } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index df7d5d25e8c..505eb11cd61 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -33,11 +33,13 @@ pub const ETH_SYNCING_TIMEOUT: Duration = Duration::from_secs(1); pub const ENGINE_NEW_PAYLOAD_V1: &str = "engine_newPayloadV1"; pub const ENGINE_NEW_PAYLOAD_V2: &str = "engine_newPayloadV2"; pub const ENGINE_NEW_PAYLOAD_V3: &str = "engine_newPayloadV3"; +pub const ENGINE_NEW_PAYLOAD_V4: &str = "engine_newPayloadV4"; pub const ENGINE_NEW_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(8); pub const ENGINE_GET_PAYLOAD_V1: &str = "engine_getPayloadV1"; pub const ENGINE_GET_PAYLOAD_V2: &str = "engine_getPayloadV2"; pub const ENGINE_GET_PAYLOAD_V3: &str = "engine_getPayloadV3"; +pub const ENGINE_GET_PAYLOAD_V4: &str = "engine_getPayloadV4"; pub const ENGINE_GET_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(2); pub const ENGINE_GET_BLOBS_BUNDLE_V1: &str = "engine_getBlobsBundleV1"; @@ -765,6 +767,14 @@ impl HttpJsonRpc { ) .await?, ), + ForkName::Eip6110 => ExecutionBlockWithTransactions::Eip6110( + self.rpc_request( + ETH_GET_BLOCK_BY_HASH, + params, + ETH_GET_BLOCK_BY_HASH_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?, + ), ForkName::Base | ForkName::Altair => { return Err(Error::UnsupportedForkVariant(format!( "called get_block_by_hash_with_txns with fork {:?}", @@ -876,9 +886,30 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V2(response).into()) } - ForkName::Base | ForkName::Altair | ForkName::Eip4844 => Err( - Error::UnsupportedForkVariant(format!("called get_payload_v2 with {}", fork_name)), - ), + ForkName::Eip4844 => { + let response: JsonGetPayloadResponseV3 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V2, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + Ok(JsonGetPayloadResponse::V3(response).into()) + } + ForkName::Eip6110 => { + let response: JsonGetPayloadResponseV4 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V2, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + Ok(JsonGetPayloadResponse::V4(response).into()) + } + ForkName::Base | ForkName::Altair => Err(Error::UnsupportedForkVariant(format!( + "called get_payload_v2 with {}", + fork_name + ))), } } @@ -920,6 +951,16 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V3(response).into()) } + ForkName::Eip6110 => { + let response: JsonGetPayloadResponseV4 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V4, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + Ok(JsonGetPayloadResponse::V4(response).into()) + } ForkName::Base | ForkName::Altair => Err(Error::UnsupportedForkVariant(format!( "called get_payload_v3 with {}", fork_name diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 0384aad4886..c028abdd0e1 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -7,7 +7,7 @@ use superstruct::superstruct; use types::blobs_sidecar::KzgCommitments; use types::{ Blobs, DepositReceipt, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, - ExecutionPayloadEip4844, ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, + ExecutionPayloadEip4844, ExecutionPayloadEip6110, ExecutionPayloadMerge, FixedVector, Transactions, Transaction, Unsigned, VariableList, Withdrawal, }; @@ -65,7 +65,7 @@ pub struct JsonPayloadIdResponse { } #[superstruct( - variants(V1, V2, V3), + variants(V1, V2, V3, V4), variant_attributes( derive(Debug, PartialEq, Default, Serialize, Deserialize,), serde(bound = "T: EthSpec", rename_all = "camelCase"), @@ -95,16 +95,16 @@ pub struct JsonExecutionPayload { pub extra_data: VariableList, #[serde(with = "eth2_serde_utils::u256_hex_be")] pub base_fee_per_gas: Uint256, - #[superstruct(only(V3))] + #[superstruct(only(V3, V4))] #[serde(with = "eth2_serde_utils::u256_hex_be")] pub excess_data_gas: Uint256, pub block_hash: ExecutionBlockHash, #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] - pub transactions: Transactions, - #[superstruct(only(V2, V3))] + pub transactions: VariableList, T::MaxTransactionsPerPayload>, + #[superstruct(only(V2, V3, V4))] pub withdrawals: VariableList, - #[superstruct(only(V3))] - pub deposit_receipts: VariableList, + #[superstruct(only(V4))] + pub deposit_receipts: VariableList, } impl From> for JsonExecutionPayloadV1 { @@ -177,6 +177,33 @@ impl From> for JsonExecutionPayloadV3 .map(Into::into) .collect::>() .into(), + } + } +} +impl From> for JsonExecutionPayloadV4 { + fn from(payload: ExecutionPayloadEip6110) -> Self { + JsonExecutionPayloadV4 { + parent_hash: payload.parent_hash, + fee_recipient: payload.fee_recipient, + state_root: payload.state_root, + receipts_root: payload.receipts_root, + logs_bloom: payload.logs_bloom, + prev_randao: payload.prev_randao, + block_number: payload.block_number, + gas_limit: payload.gas_limit, + gas_used: payload.gas_used, + timestamp: payload.timestamp, + extra_data: payload.extra_data, + base_fee_per_gas: payload.base_fee_per_gas, + excess_data_gas: payload.excess_data_gas, + block_hash: payload.block_hash, + transactions: payload.transactions, + withdrawals: payload + .withdrawals + .into_iter() + .map(Into::into) + .collect::>() + .into(), deposit_receipts: payload .deposit_receipts .into_iter() @@ -193,6 +220,7 @@ impl From> for JsonExecutionPayload { ExecutionPayload::Merge(payload) => JsonExecutionPayload::V1(payload.into()), ExecutionPayload::Capella(payload) => JsonExecutionPayload::V2(payload.into()), ExecutionPayload::Eip4844(payload) => JsonExecutionPayload::V3(payload.into()), + ExecutionPayload::Eip6110(payload) => JsonExecutionPayload::V4(payload.into()), } } } @@ -267,6 +295,33 @@ impl From> for ExecutionPayloadEip4844 .map(Into::into) .collect::>() .into(), + } + } +} +impl From> for ExecutionPayloadEip6110 { + fn from(payload: JsonExecutionPayloadV4) -> Self { + ExecutionPayloadEip6110 { + parent_hash: payload.parent_hash, + fee_recipient: payload.fee_recipient, + state_root: payload.state_root, + receipts_root: payload.receipts_root, + logs_bloom: payload.logs_bloom, + prev_randao: payload.prev_randao, + block_number: payload.block_number, + gas_limit: payload.gas_limit, + gas_used: payload.gas_used, + timestamp: payload.timestamp, + extra_data: payload.extra_data, + base_fee_per_gas: payload.base_fee_per_gas, + excess_data_gas: payload.excess_data_gas, + block_hash: payload.block_hash, + transactions: payload.transactions, + withdrawals: payload + .withdrawals + .into_iter() + .map(Into::into) + .collect::>() + .into(), deposit_receipts: payload .deposit_receipts .into_iter() @@ -283,12 +338,13 @@ impl From> for ExecutionPayload { JsonExecutionPayload::V1(payload) => ExecutionPayload::Merge(payload.into()), JsonExecutionPayload::V2(payload) => ExecutionPayload::Capella(payload.into()), JsonExecutionPayload::V3(payload) => ExecutionPayload::Eip4844(payload.into()), + JsonExecutionPayload::V4(payload) => ExecutionPayload::Eip6110(payload.into()), } } } #[superstruct( - variants(V1, V2, V3), + variants(V1, V2, V3, V4), variant_attributes( derive(Debug, PartialEq, Serialize, Deserialize), serde(bound = "T: EthSpec", rename_all = "camelCase") @@ -305,6 +361,8 @@ pub struct JsonGetPayloadResponse { pub execution_payload: JsonExecutionPayloadV2, #[superstruct(only(V3), partial_getter(rename = "execution_payload_v3"))] pub execution_payload: JsonExecutionPayloadV3, + #[superstruct(only(V4), partial_getter(rename = "execution_payload_v4"))] + pub execution_payload: JsonExecutionPayloadV4, #[serde(with = "eth2_serde_utils::u256_hex_be")] pub block_value: Uint256, } @@ -330,6 +388,12 @@ impl From> for GetPayloadResponse { block_value: response.block_value, }) } + JsonGetPayloadResponse::V4(response) => { + GetPayloadResponse::Eip6110(GetPayloadResponseEip6110 { + execution_payload: response.execution_payload.into(), + block_value: response.block_value, + }) + } } } } @@ -422,6 +486,8 @@ pub struct JsonPayloadAttributes { pub suggested_fee_recipient: Address, #[superstruct(only(V2))] pub withdrawals: Vec, + #[superstruct(only(V2))] + pub deposit_receipts: Vec, } impl From for JsonPayloadAttributes { @@ -437,6 +503,7 @@ impl From for JsonPayloadAttributes { prev_randao: pa.prev_randao, suggested_fee_recipient: pa.suggested_fee_recipient, withdrawals: pa.withdrawals.into_iter().map(Into::into).collect(), + deposit_receipts: pa.deposit_receipts.into_iter().map(Into::into).collect(), }), } } @@ -455,6 +522,7 @@ impl From for PayloadAttributes { prev_randao: jpa.prev_randao, suggested_fee_recipient: jpa.suggested_fee_recipient, withdrawals: jpa.withdrawals.into_iter().map(Into::into).collect(), + deposit_receipts: jpa.deposit_receipts.into_iter().map(Into::into).collect(), }), } } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 9702e210569..4a7ea6ccdb7 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -43,13 +43,14 @@ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::consts::eip4844::BLOB_TX_TYPE; use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlobTransaction}; +use types::Withdrawals; use types::{ blobs_sidecar::{Blobs, KzgCommitments}, BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, - ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge, ForkName, + ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadEip6110, + ExecutionPayloadMerge, ForkName, }; use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; -use types::{DepositReceipt, Withdrawals}; use types::{ ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Transaction, Uint256, @@ -213,6 +214,12 @@ impl> BlockProposalContents BlockProposalContents::PayloadAndBlobs { + payload: Payload::default_at_fork(fork_name)?, + block_value: Uint256::zero(), + blobs: VariableList::default(), + kzg_commitments: VariableList::default(), + }, }) } } @@ -1110,7 +1117,7 @@ impl ExecutionLayer { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { None } - ForkName::Eip4844 => { + ForkName::Eip4844 | ForkName::Eip6110 => { debug!( self.log(), "Issuing engine_getBlobsBundle"; @@ -1703,6 +1710,7 @@ impl ExecutionLayer { ForkName::Merge => Ok(Some(ExecutionPayloadMerge::default().into())), ForkName::Capella => Ok(Some(ExecutionPayloadCapella::default().into())), ForkName::Eip4844 => Ok(Some(ExecutionPayloadEip4844::default().into())), + ForkName::Eip6110 => Ok(Some(ExecutionPayloadEip6110::default().into())), ForkName::Base | ForkName::Altair => Err(ApiError::UnsupportedForkVariant( format!("called get_payload_by_block_hash_from_engine with {}", fork), )), @@ -1785,15 +1793,6 @@ impl ExecutionLayer { ) .map_err(ApiError::DeserializeWithdrawals)?; - let deposit_receipts = eip4844_block - .deposit_receipts - .into_iter() - .map(Into::into) - .collect::>(); - - let deposit_receipts = VariableList::new(deposit_receipts) - .map_err(ApiError::DeserializeDepositReceipts)?; - ExecutionPayload::Eip4844(ExecutionPayloadEip4844 { parent_hash: eip4844_block.parent_hash, fee_recipient: eip4844_block.fee_recipient, @@ -1811,6 +1810,44 @@ impl ExecutionLayer { block_hash: eip4844_block.block_hash, transactions: convert_transactions(eip4844_block.transactions)?, withdrawals, + }) + } + ExecutionBlockWithTransactions::Eip6110(eip6110_block) => { + let withdrawals = VariableList::new( + eip6110_block + .withdrawals + .into_iter() + .map(Into::into) + .collect(), + ) + .map_err(ApiError::DeserializeWithdrawals)?; + + let deposit_receipts = VariableList::new( + eip6110_block + .deposit_receipts + .into_iter() + .map(Into::into) + .collect(), + ) + .map_err(ApiError::DeserializeDepositReceipts)?; + + ExecutionPayload::Eip6110(ExecutionPayloadEip6110 { + parent_hash: eip6110_block.parent_hash, + fee_recipient: eip6110_block.fee_recipient, + state_root: eip6110_block.state_root, + receipts_root: eip6110_block.receipts_root, + logs_bloom: eip6110_block.logs_bloom, + prev_randao: eip6110_block.prev_randao, + block_number: eip6110_block.block_number, + gas_limit: eip6110_block.gas_limit, + gas_used: eip6110_block.gas_used, + timestamp: eip6110_block.timestamp, + extra_data: eip6110_block.extra_data, + base_fee_per_gas: eip6110_block.base_fee_per_gas, + excess_data_gas: eip6110_block.excess_data_gas, + block_hash: eip6110_block.block_hash, + transactions: convert_transactions(eip6110_block.transactions)?, + withdrawals, deposit_receipts, }) } diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index a6db7eee594..768c2d35dc5 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -14,7 +14,8 @@ use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use types::{ EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, - ExecutionPayloadEip4844, ExecutionPayloadMerge, ForkName, Hash256, Uint256, + ExecutionPayloadEip4844, ExecutionPayloadEip6110, ExecutionPayloadMerge, ForkName, Hash256, + Uint256, }; const GAS_LIMIT: u64 = 16384; @@ -119,6 +120,7 @@ pub struct ExecutionBlockGenerator { */ pub shanghai_time: Option, // withdrawals pub eip4844_time: Option, // 4844 + pub eip6110_time: Option, // 6110 } impl ExecutionBlockGenerator { @@ -128,6 +130,7 @@ impl ExecutionBlockGenerator { terminal_block_hash: ExecutionBlockHash, shanghai_time: Option, eip4844_time: Option, + eip6110_time: Option, ) -> Self { let mut gen = Self { head_block: <_>::default(), @@ -142,6 +145,7 @@ impl ExecutionBlockGenerator { payload_ids: <_>::default(), shanghai_time, eip4844_time, + eip6110_time, }; gen.insert_pow_block(0).unwrap(); @@ -174,11 +178,14 @@ impl ExecutionBlockGenerator { } pub fn get_fork_at_timestamp(&self, timestamp: u64) -> ForkName { - match self.eip4844_time { - Some(fork_time) if timestamp >= fork_time => ForkName::Eip4844, - _ => match self.shanghai_time { - Some(fork_time) if timestamp >= fork_time => ForkName::Capella, - _ => ForkName::Merge, + match self.eip6110_time { + Some(fork_time) if timestamp >= fork_time => ForkName::Eip6110, + _ => match self.eip4844_time { + Some(fork_time) if timestamp >= fork_time => ForkName::Eip4844, + _ => match self.shanghai_time { + Some(fork_time) if timestamp >= fork_time => ForkName::Capella, + _ => ForkName::Merge, + }, }, } } @@ -554,7 +561,28 @@ impl ExecutionBlockGenerator { block_hash: ExecutionBlockHash::zero(), transactions: vec![].into(), withdrawals: pa.withdrawals.clone().into(), - deposit_receipts: vec![].into(), // TODO: Check if deposit receipts should be part of PayloadAttributesV2 + }) + } + ForkName::Eip6110 => { + ExecutionPayload::Eip6110(ExecutionPayloadEip6110 { + parent_hash: forkchoice_state.head_block_hash, + fee_recipient: pa.suggested_fee_recipient, + receipts_root: Hash256::repeat_byte(42), + state_root: Hash256::repeat_byte(43), + logs_bloom: vec![0; 256].into(), + prev_randao: pa.prev_randao, + block_number: parent.block_number() + 1, + gas_limit: GAS_LIMIT, + gas_used: GAS_USED, + timestamp: pa.timestamp, + extra_data: "block gen was here".as_bytes().to_vec().into(), + base_fee_per_gas: Uint256::one(), + // FIXME(4844): maybe this should be set to something? + excess_data_gas: Uint256::one(), + block_hash: ExecutionBlockHash::zero(), + transactions: vec![].into(), + withdrawals: pa.withdrawals.clone().into(), + deposit_receipts: pa.deposit_receipts.clone().into(), }) } _ => unreachable!(), @@ -651,6 +679,7 @@ mod test { ExecutionBlockHash::zero(), None, None, + None, ); for i in 0..=TERMINAL_BLOCK { diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index a1488e2dc9a..3c483bdb8fb 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -112,6 +112,21 @@ pub async fn handle_rpc( }) }) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, + ENGINE_NEW_PAYLOAD_V4 => get_param::>(params, 0) + .map(|jep| JsonExecutionPayload::V4(jep)) + .or_else(|_| { + get_param::>(params, 0) + .map(|jep| JsonExecutionPayload::V3(jep)) + .or_else(|_| { + get_param::>(params, 0) + .map(|jep| JsonExecutionPayload::V2(jep)) + .or_else(|_| { + get_param::>(params, 0) + .map(|jep| JsonExecutionPayload::V1(jep)) + }) + }) + }) + .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, _ => unreachable!(), }; @@ -175,6 +190,44 @@ pub async fn handle_rpc( )); } } + ForkName::Eip6110 => { + if method == ENGINE_NEW_PAYLOAD_V1 + || method == ENGINE_NEW_PAYLOAD_V2 + || method == ENGINE_NEW_PAYLOAD_V3 + { + return Err(( + format!("{} called after eip6110 fork!", method), + GENERIC_ERROR_CODE, + )); + } + if matches!(request, JsonExecutionPayload::V1(_)) { + return Err(( + format!( + "{} called with `ExecutionPayloadV1` after eip6110 fork!", + method + ), + GENERIC_ERROR_CODE, + )); + } + if matches!(request, JsonExecutionPayload::V2(_)) { + return Err(( + format!( + "{} called with `ExecutionPayloadV2` after eip6110 fork!", + method + ), + GENERIC_ERROR_CODE, + )); + } + if matches!(request, JsonExecutionPayload::V3(_)) { + return Err(( + format!( + "{} called with `ExecutionPayloadV3` after eip6110 fork!", + method + ), + GENERIC_ERROR_CODE, + )); + } + } _ => unreachable!(), }; @@ -208,7 +261,10 @@ pub async fn handle_rpc( Ok(serde_json::to_value(JsonPayloadStatusV1::from(response)).unwrap()) } - ENGINE_GET_PAYLOAD_V1 | ENGINE_GET_PAYLOAD_V2 | ENGINE_GET_PAYLOAD_V3 => { + ENGINE_GET_PAYLOAD_V1 + | ENGINE_GET_PAYLOAD_V2 + | ENGINE_GET_PAYLOAD_V3 + | ENGINE_GET_PAYLOAD_V4 => { let request: JsonPayloadIdRequest = get_param(params, 0).map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?; let id = request.into(); @@ -294,6 +350,43 @@ pub async fn handle_rpc( }) .unwrap() } + JsonExecutionPayload::V4(execution_payload) => { + serde_json::to_value(JsonGetPayloadResponseV4 { + execution_payload, + block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), + }) + .unwrap() + } + }), + ENGINE_GET_PAYLOAD_V4 => Ok(match JsonExecutionPayload::from(response) { + JsonExecutionPayload::V1(execution_payload) => { + serde_json::to_value(JsonGetPayloadResponseV1 { + execution_payload, + block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), + }) + .unwrap() + } + JsonExecutionPayload::V2(execution_payload) => { + serde_json::to_value(JsonGetPayloadResponseV2 { + execution_payload, + block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), + }) + .unwrap() + } + JsonExecutionPayload::V3(execution_payload) => { + serde_json::to_value(JsonGetPayloadResponseV3 { + execution_payload, + block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), + }) + .unwrap() + } + JsonExecutionPayload::V4(execution_payload) => { + serde_json::to_value(JsonGetPayloadResponseV4 { + execution_payload, + block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), + }) + .unwrap() + } }), _ => unreachable!(), } diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 19972650139..b5a500c3f3e 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -405,7 +405,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { let payload_attributes = match fork { ForkName::Merge => PayloadAttributes::new(timestamp, *prev_randao, fee_recipient, None), // the withdrawals root is filled in by operations - ForkName::Capella | ForkName::Eip4844 => { + ForkName::Capella | ForkName::Eip4844 | ForkName::Eip6110 => { PayloadAttributes::new(timestamp, *prev_randao, fee_recipient, Some(vec![])) } ForkName::Base | ForkName::Altair => { @@ -452,7 +452,7 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { value: to_ssz_rs(&Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI))?, public_key: self.builder_sk.public_key(), }), - ForkName::Base | ForkName::Altair | ForkName::Eip4844 => { + ForkName::Base | ForkName::Altair | ForkName::Eip4844 | ForkName::Eip6110 => { return Err(BlindedBlockProviderError::Custom(format!( "Unsupported fork: {}", fork diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 1a5d1fd1983..8848607856d 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -30,6 +30,7 @@ impl MockExecutionLayer { None, None, None, + None, Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec, None, @@ -42,6 +43,7 @@ impl MockExecutionLayer { terminal_block: u64, shanghai_time: Option, eip4844_time: Option, + eip6110_time: Option, builder_threshold: Option, jwt_key: Option, spec: ChainSpec, @@ -58,6 +60,7 @@ impl MockExecutionLayer { spec.terminal_block_hash, shanghai_time, eip4844_time, + eip6110_time, ); let url = SensitiveUrl::parse(&server.url()).unwrap(); diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 3c0763a8fb9..e1a148caaa5 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -63,6 +63,7 @@ pub struct MockExecutionConfig { pub terminal_block_hash: ExecutionBlockHash, pub shanghai_time: Option, pub eip4844_time: Option, + pub eip6110_time: Option, } impl Default for MockExecutionConfig { @@ -75,6 +76,7 @@ impl Default for MockExecutionConfig { server_config: Config::default(), shanghai_time: None, eip4844_time: None, + eip6110_time: None, } } } @@ -96,6 +98,7 @@ impl MockServer { ExecutionBlockHash::zero(), None, // FIXME(capella): should this be the default? None, // FIXME(eip4844): should this be the default? + None, // FIXME(eip6110): should this be the default? ) } @@ -108,6 +111,7 @@ impl MockServer { server_config, shanghai_time, eip4844_time, + eip6110_time, } = config; let last_echo_request = Arc::new(RwLock::new(None)); let preloaded_responses = Arc::new(Mutex::new(vec![])); @@ -117,6 +121,7 @@ impl MockServer { terminal_block_hash, shanghai_time, eip4844_time, + eip6110_time, ); let ctx: Arc> = Arc::new(Context { @@ -176,6 +181,7 @@ impl MockServer { terminal_block_hash: ExecutionBlockHash, shanghai_time: Option, eip4844_time: Option, + eip6110_time: Option, ) -> Self { Self::new_with_config( handle, @@ -187,6 +193,7 @@ impl MockServer { terminal_block_hash, shanghai_time, eip4844_time, + eip6110_time, }, ) } diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 79c8b67d75a..3ac41a8143e 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -412,7 +412,11 @@ pub fn gossipsub_config(network_load: u8, fork_context: Arc) -> Gos match fork_context.current_fork() { // according to: https://github.com/ethereum/consensus-specs/blob/dev/specs/merge/p2p-interface.md#the-gossip-domain-gossipsub // the derivation of the message-id remains the same in the merge and for eip 4844. - ForkName::Altair | ForkName::Merge | ForkName::Capella | ForkName::Eip4844 => { + ForkName::Altair + | ForkName::Merge + | ForkName::Capella + | ForkName::Eip4844 + | ForkName::Eip6110 => { let topic_len_bytes = topic_bytes.len().to_le_bytes(); let mut vec = Vec::with_capacity( prefix.len() + topic_len_bytes.len() + topic_bytes.len() + message.data.len(), diff --git a/beacon_node/lighthouse_network/src/rpc/codec/base.rs b/beacon_node/lighthouse_network/src/rpc/codec/base.rs index 164a7c025d9..f442e59b85e 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/base.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/base.rs @@ -195,11 +195,13 @@ mod tests { let merge_fork_epoch = Epoch::new(2); let capella_fork_epoch = Epoch::new(3); let eip4844_fork_epoch = Epoch::new(4); + let eip6110_fork_epoch = Epoch::new(5); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); chain_spec.bellatrix_fork_epoch = Some(merge_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); chain_spec.eip4844_fork_epoch = Some(eip4844_fork_epoch); + chain_spec.eip6110_fork_epoch = Some(eip6110_fork_epoch); let current_slot = match fork_name { ForkName::Base => Slot::new(0), @@ -207,6 +209,7 @@ mod tests { ForkName::Merge => merge_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Capella => capella_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Eip4844 => eip4844_fork_epoch.start_slot(Spec::slots_per_epoch()), + ForkName::Eip6110 => eip6110_fork_epoch.start_slot(Spec::slots_per_epoch()), }; ForkContext::new::(current_slot, Hash256::zero(), &chain_spec) } diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index d94e8f52218..e75758f8974 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -19,7 +19,8 @@ use types::light_client_bootstrap::LightClientBootstrap; use types::{ BlobsSidecar, EthSpec, ForkContext, ForkName, Hash256, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockAndBlobsSidecar, SignedBeaconBlockBase, - SignedBeaconBlockCapella, SignedBeaconBlockEip4844, SignedBeaconBlockMerge, + SignedBeaconBlockCapella, SignedBeaconBlockEip4844, SignedBeaconBlockEip6110, + SignedBeaconBlockMerge, }; use unsigned_varint::codec::Uvi; @@ -419,6 +420,10 @@ fn context_bytes( return match **ref_box_block { // NOTE: If you are adding another fork type here, be sure to modify the // `fork_context.to_context_bytes()` function to support it as well! + SignedBeaconBlock::Eip6110 { .. } => { + // Eip6110 context being `None` implies that "merge never happened". + fork_context.to_context_bytes(ForkName::Eip6110) + } SignedBeaconBlock::Eip4844 { .. } => { // Eip4844 context being `None` implies that "merge never happened". fork_context.to_context_bytes(ForkName::Eip4844) @@ -668,6 +673,11 @@ fn handle_v2_response( decoded_buffer, )?), )))), + ForkName::Eip6110 => Ok(Some(RPCResponse::BlocksByRange(Arc::new( + SignedBeaconBlock::Eip6110(SignedBeaconBlockEip6110::from_ssz_bytes( + decoded_buffer, + )?), + )))), }, Protocol::BlocksByRoot => match fork_name { ForkName::Altair => Ok(Some(RPCResponse::BlocksByRoot(Arc::new( @@ -693,6 +703,11 @@ fn handle_v2_response( decoded_buffer, )?), )))), + ForkName::Eip6110 => Ok(Some(RPCResponse::BlocksByRoot(Arc::new( + SignedBeaconBlock::Eip6110(SignedBeaconBlockEip6110::from_ssz_bytes( + decoded_buffer, + )?), + )))), }, Protocol::BlobsByRange => { Err(RPCError::InvalidData("blobs by range via v2".to_string())) @@ -754,11 +769,13 @@ mod tests { let merge_fork_epoch = Epoch::new(2); let capella_fork_epoch = Epoch::new(3); let eip4844_fork_epoch = Epoch::new(4); + let eip6110_fork_epoch = Epoch::new(5); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); chain_spec.bellatrix_fork_epoch = Some(merge_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); chain_spec.eip4844_fork_epoch = Some(eip4844_fork_epoch); + chain_spec.eip6110_fork_epoch = Some(eip6110_fork_epoch); let current_slot = match fork_name { ForkName::Base => Slot::new(0), @@ -766,6 +783,7 @@ mod tests { ForkName::Merge => merge_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Capella => capella_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Eip4844 => eip4844_fork_epoch.start_slot(Spec::slots_per_epoch()), + ForkName::Eip6110 => eip6110_fork_epoch.start_slot(Spec::slots_per_epoch()), }; ForkContext::new::(current_slot, Hash256::zero(), &chain_spec) } diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index d11240ebf9e..1382f498a6c 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -90,6 +90,12 @@ lazy_static! { + (::ssz_fixed_len() * ::max_blobs_per_block()) + ssz::BYTES_PER_LENGTH_OFFSET; // Length offset for the blob commitments field. + pub static ref SIGNED_BEACON_BLOCK_EIP6110_MAX: usize = *SIGNED_BEACON_BLOCK_CAPELLA_MAX_WITHOUT_PAYLOAD + + types::ExecutionPayload::::max_execution_payload_eip6110_size() // adding max size of execution payload (~16gb) + + ssz::BYTES_PER_LENGTH_OFFSET // Adding the additional offsets for the `ExecutionPayload` + + (::ssz_fixed_len() * ::max_blobs_per_block()) + + ssz::BYTES_PER_LENGTH_OFFSET; // Length offset for the blob commitments field. + pub static ref BLOCKS_BY_ROOT_REQUEST_MIN: usize = VariableList::::from(Vec::::new()) .as_ssz_bytes() @@ -130,6 +136,7 @@ pub(crate) const MAX_RPC_SIZE_POST_MERGE: usize = 10 * 1_048_576; // 10M pub(crate) const MAX_RPC_SIZE_POST_CAPELLA: usize = 10 * 1_048_576; // 10M // FIXME(sean) should this be increased to account for blobs? pub(crate) const MAX_RPC_SIZE_POST_EIP4844: usize = 10 * 1_048_576; // 10M +pub(crate) const MAX_RPC_SIZE_POST_EIP6110: usize = 10 * 1_048_576; // 10M /// The protocol prefix the RPC protocol id. const PROTOCOL_PREFIX: &str = "/eth2/beacon_chain/req"; /// Time allowed for the first byte of a request to arrive before we time out (Time To First Byte). @@ -145,6 +152,7 @@ pub fn max_rpc_size(fork_context: &ForkContext) -> usize { ForkName::Merge => MAX_RPC_SIZE_POST_MERGE, ForkName::Capella => MAX_RPC_SIZE_POST_CAPELLA, ForkName::Eip4844 => MAX_RPC_SIZE_POST_EIP4844, + ForkName::Eip6110 => MAX_RPC_SIZE_POST_EIP6110, } } @@ -173,6 +181,10 @@ pub fn rpc_block_limits_by_fork(current_fork: ForkName) -> RpcLimits { *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks *SIGNED_BEACON_BLOCK_EIP4844_MAX, // EIP 4844 block is larger than all prior fork blocks ), + ForkName::Eip6110 => RpcLimits::new( + *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks + *SIGNED_BEACON_BLOCK_EIP6110_MAX, // EIP 6110 block is larger than all prior fork blocks + ), } } diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 7951a072438..4b47a8641ed 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -190,6 +190,12 @@ impl PubsubMessage { .to_string(), ) } + Some(ForkName::Eip6110) => { + return Err( + "beacon_block topic is not used from eip4844 fork onwards" + .to_string(), + ) + } Some(ForkName::Capella) => SignedBeaconBlock::::Capella( SignedBeaconBlockCapella::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, @@ -205,7 +211,7 @@ impl PubsubMessage { } GossipKind::BeaconBlocksAndBlobsSidecar => { match fork_context.from_context_bytes(gossip_topic.fork_digest) { - Some(ForkName::Eip4844) => { + Some(ForkName::Eip4844) | Some(ForkName::Eip6110) => { let block_and_blobs_sidecar = SignedBeaconBlockAndBlobsSidecar::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?; diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 20f836b76a1..a78f21af9cd 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -48,6 +48,7 @@ pub fn fork_core_topics(fork_name: &ForkName) -> Vec { ForkName::Merge => vec![], ForkName::Capella => CAPELLA_CORE_TOPICS.to_vec(), ForkName::Eip4844 => vec![], // TODO + ForkName::Eip6110 => vec![], // TODO } } diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index bd153284edb..740b5a151b0 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -27,11 +27,13 @@ pub fn fork_context(fork_name: ForkName) -> ForkContext { let merge_fork_epoch = Epoch::new(2); let capella_fork_epoch = Epoch::new(3); let eip4844_fork_epoch = Epoch::new(4); + let eip6110_fork_epoch = Epoch::new(5); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); chain_spec.bellatrix_fork_epoch = Some(merge_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); chain_spec.eip4844_fork_epoch = Some(eip4844_fork_epoch); + chain_spec.eip6110_fork_epoch = Some(eip6110_fork_epoch); let current_slot = match fork_name { ForkName::Base => Slot::new(0), @@ -39,6 +41,7 @@ pub fn fork_context(fork_name: ForkName) -> ForkContext { ForkName::Merge => merge_fork_epoch.start_slot(E::slots_per_epoch()), ForkName::Capella => capella_fork_epoch.start_slot(E::slots_per_epoch()), ForkName::Eip4844 => eip4844_fork_epoch.start_slot(E::slots_per_epoch()), + ForkName::Eip6110 => eip6110_fork_epoch.start_slot(E::slots_per_epoch()), }; ForkContext::new::(current_slot, Hash256::zero(), &chain_spec) } diff --git a/beacon_node/store/src/impls/execution_payload.rs b/beacon_node/store/src/impls/execution_payload.rs index 01a2dba0b0a..d89fe015617 100644 --- a/beacon_node/store/src/impls/execution_payload.rs +++ b/beacon_node/store/src/impls/execution_payload.rs @@ -2,7 +2,7 @@ use crate::{DBColumn, Error, StoreItem}; use ssz::{Decode, Encode}; use types::{ BlobsSidecar, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, - ExecutionPayloadMerge, + ExecutionPayloadEip6110, ExecutionPayloadMerge, }; macro_rules! impl_store_item { @@ -25,6 +25,7 @@ macro_rules! impl_store_item { impl_store_item!(ExecutionPayloadMerge); impl_store_item!(ExecutionPayloadCapella); impl_store_item!(ExecutionPayloadEip4844); +impl_store_item!(ExecutionPayloadEip6110); impl_store_item!(BlobsSidecar); /// This fork-agnostic implementation should be only used for writing. diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index b642c2752c6..e84e3409fb7 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -15,7 +15,7 @@ use types::*; /// /// Utilises lazy-loading from separate storage for its vector fields. #[superstruct( - variants(Base, Altair, Merge, Capella, Eip4844), + variants(Base, Altair, Merge, Capella, Eip4844, Eip6110), variant_attributes(derive(Debug, PartialEq, Clone, Encode, Decode)) )] #[derive(Debug, PartialEq, Clone, Encode)] @@ -67,9 +67,9 @@ where pub current_epoch_attestations: VariableList, T::MaxPendingAttestations>, // Participation (Altair and later) - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub previous_epoch_participation: VariableList, - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub current_epoch_participation: VariableList, // Finality @@ -79,13 +79,13 @@ where pub finalized_checkpoint: Checkpoint, // Inactivity - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub inactivity_scores: VariableList, // Light-client sync committees - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub current_sync_committee: Arc>, - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub next_sync_committee: Arc>, // Execution @@ -104,18 +104,23 @@ where partial_getter(rename = "latest_execution_payload_header_eip4844") )] pub latest_execution_payload_header: ExecutionPayloadHeaderEip4844, + #[superstruct( + only(Eip6110), + partial_getter(rename = "latest_execution_payload_header_eip6110") + )] + pub latest_execution_payload_header: ExecutionPayloadHeaderEip6110, // Capella - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Eip4844, Eip6110))] pub next_withdrawal_index: u64, - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Eip4844, Eip6110))] pub next_withdrawal_validator_index: u64, #[ssz(skip_serializing, skip_deserializing)] - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Eip4844, Eip6110))] pub historical_summaries: Option>, - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Eip6110))] pub deposit_receipts_start_index: u64, } @@ -226,8 +231,7 @@ impl PartialBeaconState { inactivity_scores, latest_execution_payload_header, next_withdrawal_index, - next_withdrawal_validator_index, - deposit_receipts_start_index + next_withdrawal_validator_index ], [historical_summaries] ), @@ -236,6 +240,23 @@ impl PartialBeaconState { outer, Eip4844, PartialBeaconStateEip4844, + [ + previous_epoch_participation, + current_epoch_participation, + current_sync_committee, + next_sync_committee, + inactivity_scores, + latest_execution_payload_header, + next_withdrawal_index, + next_withdrawal_validator_index + ], + [historical_summaries] + ), + BeaconState::Eip6110(s) => impl_from_state_forgetful!( + s, + outer, + Eip6110, + PartialBeaconStateEip6110, [ previous_epoch_participation, current_epoch_participation, @@ -473,8 +494,7 @@ impl TryInto> for PartialBeaconState { inactivity_scores, latest_execution_payload_header, next_withdrawal_index, - next_withdrawal_validator_index, - deposit_receipts_start_index + next_withdrawal_validator_index ], [historical_summaries] ), @@ -482,6 +502,22 @@ impl TryInto> for PartialBeaconState { inner, Eip4844, BeaconStateEip4844, + [ + previous_epoch_participation, + current_epoch_participation, + current_sync_committee, + next_sync_committee, + inactivity_scores, + latest_execution_payload_header, + next_withdrawal_index, + next_withdrawal_validator_index + ], + [historical_summaries] + ), + PartialBeaconState::Eip6110(inner) => impl_try_into_beacon_state!( + inner, + Eip6110, + BeaconStateEip6110, [ previous_epoch_participation, current_epoch_participation, diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index d14746551c3..db07444e48e 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -959,9 +959,11 @@ impl ForkVersionDeserialize for SsePayloadAttributes { ForkName::Merge => serde_json::from_value(value) .map(Self::V1) .map_err(serde::de::Error::custom), - ForkName::Capella | ForkName::Eip4844 => serde_json::from_value(value) - .map(Self::V2) - .map_err(serde::de::Error::custom), + ForkName::Capella | ForkName::Eip4844 | ForkName::Eip6110 => { + serde_json::from_value(value) + .map(Self::V2) + .map_err(serde::de::Error::custom) + } ForkName::Base | ForkName::Altair => Err(serde::de::Error::custom(format!( "SsePayloadAttributes deserialization for {fork_name} not implemented" ))), diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml b/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml index f7334b6187c..ecd9ad80cc8 100644 --- a/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml @@ -38,6 +38,10 @@ CAPELLA_FORK_EPOCH: 1 EIP4844_FORK_VERSION: 0x50484404 EIP4844_FORK_EPOCH: 5 +# EIP6110 +EIP6110_FORK_VERSION: 0x60484404 +EIP6110_FORK_EPOCH: 10 + # Time parameters # --------------------------------------------------------------- # 12 seconds diff --git a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml index 6aa2c9590a5..0386babb800 100644 --- a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml @@ -42,6 +42,9 @@ CAPELLA_FORK_EPOCH: 18446744073709551615 # Eip4844 EIP4844_FORK_VERSION: 0x04000064 EIP4844_FORK_EPOCH: 18446744073709551615 +# Eip6110 +EIP6110_FORK_VERSION: 0x05000064 +EIP6110_FORK_EPOCH: 18446744073709551615 # Sharding SHARDING_FORK_VERSION: 0x03000064 SHARDING_FORK_EPOCH: 18446744073709551615 diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml index b5ce752b700..7fddf476f99 100644 --- a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml @@ -42,6 +42,9 @@ CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC # Eip4844 EIP4844_FORK_VERSION: 0x04000000 EIP4844_FORK_EPOCH: 18446744073709551615 +# Eip6110 +EIP6110_FORK_VERSION: 0x05000000 +EIP6110_FORK_EPOCH: 18446744073709551615 # Sharding SHARDING_FORK_VERSION: 0x03000000 SHARDING_FORK_EPOCH: 18446744073709551615 diff --git a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml index 4ba006ec945..95a2eaf0d28 100644 --- a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml @@ -36,6 +36,10 @@ CAPELLA_FORK_EPOCH: 56832 EIP4844_FORK_VERSION: 0x03001020 EIP4844_FORK_EPOCH: 18446744073709551615 +# Eip6110 +EIP6110_FORK_VERSION: 0x04001020 +EIP6110_FORK_EPOCH: 18446744073709551615 + # Sharding SHARDING_FORK_VERSION: 0x04001020 SHARDING_FORK_EPOCH: 18446744073709551615 diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index d717c31aaf8..cf8407934c5 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -754,6 +754,7 @@ where // TODO(eip4844): Ensure that the final specification // does not substantially modify per epoch processing. BeaconBlockRef::Eip4844(_) + | BeaconBlockRef::Eip6110(_) | BeaconBlockRef::Capella(_) | BeaconBlockRef::Merge(_) | BeaconBlockRef::Altair(_) => { diff --git a/consensus/state_processing/src/common/slash_validator.rs b/consensus/state_processing/src/common/slash_validator.rs index 77cd1a32659..c53b9cbaacb 100644 --- a/consensus/state_processing/src/common/slash_validator.rs +++ b/consensus/state_processing/src/common/slash_validator.rs @@ -56,6 +56,9 @@ pub fn slash_validator( | BeaconState::Eip4844(_) => whistleblower_reward .safe_mul(PROPOSER_WEIGHT)? .safe_div(WEIGHT_DENOMINATOR)?, + BeaconState::Eip6110(_) => whistleblower_reward + .safe_mul(PROPOSER_WEIGHT)? + .safe_div(WEIGHT_DENOMINATOR)?, }; // Ensure the whistleblower index is in the validator registry. diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 275b214c8f7..10b9577400b 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -68,7 +68,12 @@ pub fn initialize_beacon_state_from_eth1( if let ExecutionPayloadHeader::Eip4844(header) = header { *header_mut = header; } - } // TODO: Should a Eip6110 or upgrade_to_eip6110() be added? + } + ExecutionPayloadHeaderRefMut::Eip6110(header_mut) => { + if let ExecutionPayloadHeader::Eip6110(header) = header { + *header_mut = header; + } + } } } diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index adea2df1c59..49ecaf23c36 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -29,6 +29,7 @@ pub use verify_exit::verify_exit; pub mod altair; pub mod block_signature_verifier; pub mod eip4844; +pub mod eip6110; pub mod errors; mod is_valid_indexed_attestation; pub mod process_operations; @@ -412,6 +413,12 @@ pub fn process_execution_payload>( _ => return Err(BlockProcessingError::IncorrectStateType), } } + ExecutionPayloadHeaderRefMut::Eip6110(header_mut) => { + match payload.to_execution_payload_header() { + ExecutionPayloadHeader::Eip6110(header) => *header_mut = header, + _ => return Err(BlockProcessingError::IncorrectStateType), + } + } } Ok(()) @@ -558,7 +565,7 @@ pub fn process_withdrawals>( ) -> Result<(), BlockProcessingError> { match state { BeaconState::Merge(_) => Ok(()), - BeaconState::Capella(_) | BeaconState::Eip4844(_) => { + BeaconState::Capella(_) | BeaconState::Eip4844(_) | BeaconState::Eip6110(_) => { let expected_withdrawals = get_expected_withdrawals(state, spec)?; let expected_root = expected_withdrawals.tree_hash_root(); let withdrawals_root = payload.withdrawals_root()?; diff --git a/consensus/state_processing/src/per_block_processing/eip6110.rs b/consensus/state_processing/src/per_block_processing/eip6110.rs new file mode 100644 index 00000000000..3358e302a60 --- /dev/null +++ b/consensus/state_processing/src/per_block_processing/eip6110.rs @@ -0,0 +1,2 @@ +#[allow(clippy::module_inception)] +pub mod eip6110; diff --git a/consensus/state_processing/src/per_block_processing/eip6110/eip6110.rs b/consensus/state_processing/src/per_block_processing/eip6110/eip6110.rs new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/consensus/state_processing/src/per_block_processing/eip6110/eip6110.rs @@ -0,0 +1 @@ + diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 903ee56ec9d..bb9fc9bf5ac 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -273,7 +273,8 @@ pub fn process_attestations>( BeaconBlockBodyRef::Altair(_) | BeaconBlockBodyRef::Merge(_) | BeaconBlockBodyRef::Capella(_) - | BeaconBlockBodyRef::Eip4844(_) => { + | BeaconBlockBodyRef::Eip4844(_) + | BeaconBlockBodyRef::Eip6110(_) => { altair::process_attestations( state, block_body.attestations(), diff --git a/consensus/state_processing/src/per_epoch_processing.rs b/consensus/state_processing/src/per_epoch_processing.rs index 996e39c27fb..8653b3541cb 100644 --- a/consensus/state_processing/src/per_epoch_processing.rs +++ b/consensus/state_processing/src/per_epoch_processing.rs @@ -14,6 +14,7 @@ pub mod altair; pub mod base; pub mod capella; pub mod effective_balance_updates; +pub mod eip4844; pub mod epoch_processing_summary; pub mod errors; pub mod historical_roots_update; @@ -41,6 +42,7 @@ pub fn process_epoch( BeaconState::Base(_) => base::process_epoch(state, spec), BeaconState::Altair(_) | BeaconState::Merge(_) => altair::process_epoch(state, spec), BeaconState::Capella(_) | BeaconState::Eip4844(_) => capella::process_epoch(state, spec), + BeaconState::Eip4844(_) | BeaconState::Eip6110(_) => capella::process_epoch(state, spec), } } diff --git a/consensus/state_processing/src/per_epoch_processing/eip4844.rs b/consensus/state_processing/src/per_epoch_processing/eip4844.rs new file mode 100644 index 00000000000..5af3758e5b6 --- /dev/null +++ b/consensus/state_processing/src/per_epoch_processing/eip4844.rs @@ -0,0 +1,75 @@ +use super::altair::inactivity_updates::process_inactivity_updates; +use super::altair::justification_and_finalization::process_justification_and_finalization; +use super::altair::participation_cache::ParticipationCache; +use super::altair::participation_flag_updates::process_participation_flag_updates; +use super::altair::rewards_and_penalties::process_rewards_and_penalties; +use super::altair::sync_committee_updates::process_sync_committee_updates; +use super::capella::process_historical_summaries_update; +use super::{process_registry_updates, process_slashings, EpochProcessingSummary, Error}; +use crate::per_epoch_processing::{ + effective_balance_updates::process_effective_balance_updates, + resets::{process_eth1_data_reset, process_randao_mixes_reset, process_slashings_reset}, +}; +use types::{BeaconState, ChainSpec, EthSpec, RelativeEpoch}; + +pub fn process_epoch( + state: &mut BeaconState, + spec: &ChainSpec, +) -> Result, Error> { + // Ensure the committee caches are built. + state.build_committee_cache(RelativeEpoch::Previous, spec)?; + state.build_committee_cache(RelativeEpoch::Current, spec)?; + state.build_committee_cache(RelativeEpoch::Next, spec)?; + + // Pre-compute participating indices and total balances. + let participation_cache = ParticipationCache::new(state, spec)?; + let sync_committee = state.current_sync_committee()?.clone(); + + // Justification and finalization. + let justification_and_finalization_state = + process_justification_and_finalization(state, &participation_cache)?; + justification_and_finalization_state.apply_changes_to_state(state); + + process_inactivity_updates(state, &participation_cache, spec)?; + + // Rewards and Penalties. + process_rewards_and_penalties(state, &participation_cache, spec)?; + + // Registry Updates. + process_registry_updates(state, spec)?; + + // Slashings. + process_slashings( + state, + participation_cache.current_epoch_total_active_balance(), + spec, + )?; + + // Reset eth1 data votes. + process_eth1_data_reset(state)?; + + // Update effective balances with hysteresis (lag). + process_effective_balance_updates(state, spec)?; + + // Reset slashings + process_slashings_reset(state)?; + + // Set randao mix + process_randao_mixes_reset(state)?; + + // Set historical summaries accumulator + process_historical_summaries_update(state)?; + + // Rotate current/previous epoch participation + process_participation_flag_updates(state)?; + + process_sync_committee_updates(state, spec)?; + + // Rotate the epoch caches to suit the epoch transition. + state.advance_caches(spec)?; + + Ok(EpochProcessingSummary::Altair { + participation_cache, + sync_committee, + }) +} diff --git a/consensus/state_processing/src/per_slot_processing.rs b/consensus/state_processing/src/per_slot_processing.rs index 8d2600bb41e..7229e2a906b 100644 --- a/consensus/state_processing/src/per_slot_processing.rs +++ b/consensus/state_processing/src/per_slot_processing.rs @@ -1,5 +1,6 @@ use crate::upgrade::{ upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_eip4844, + upgrade_to_eip6110, }; use crate::{per_epoch_processing::EpochProcessingSummary, *}; use safe_arith::{ArithError, SafeArith}; @@ -65,6 +66,10 @@ pub fn per_slot_processing( if spec.eip4844_fork_epoch == Some(state.current_epoch()) { upgrade_to_eip4844(state, spec)?; } + // Eip6110 + if spec.eip6110_fork_epoch == Some(state.current_epoch()) { + upgrade_to_eip6110(state, spec)?; + } } Ok(summary) diff --git a/consensus/state_processing/src/upgrade.rs b/consensus/state_processing/src/upgrade.rs index 01b65710564..5e7f4a9e5cf 100644 --- a/consensus/state_processing/src/upgrade.rs +++ b/consensus/state_processing/src/upgrade.rs @@ -1,9 +1,11 @@ pub mod altair; pub mod capella; pub mod eip4844; +pub mod eip6110; pub mod merge; pub use altair::upgrade_to_altair; pub use capella::upgrade_to_capella; pub use eip4844::upgrade_to_eip4844; +pub use eip6110::upgrade_to_eip6110; pub use merge::upgrade_to_bellatrix; diff --git a/consensus/state_processing/src/upgrade/capella.rs b/consensus/state_processing/src/upgrade/capella.rs index babea82cc2e..3b933fac37a 100644 --- a/consensus/state_processing/src/upgrade/capella.rs +++ b/consensus/state_processing/src/upgrade/capella.rs @@ -66,8 +66,6 @@ pub fn upgrade_to_capella( pubkey_cache: mem::take(&mut pre.pubkey_cache), exit_cache: mem::take(&mut pre.exit_cache), tree_hash_cache: mem::take(&mut pre.tree_hash_cache), - // EIP-6110 - deposit_receipts_start_index: 0, }); *pre_state = post; diff --git a/consensus/state_processing/src/upgrade/eip4844.rs b/consensus/state_processing/src/upgrade/eip4844.rs index c6e8c15c185..4f6ff9d1943 100644 --- a/consensus/state_processing/src/upgrade/eip4844.rs +++ b/consensus/state_processing/src/upgrade/eip4844.rs @@ -67,7 +67,6 @@ pub fn upgrade_to_eip4844( pubkey_cache: mem::take(&mut pre.pubkey_cache), exit_cache: mem::take(&mut pre.exit_cache), tree_hash_cache: mem::take(&mut pre.tree_hash_cache), - deposit_receipts_start_index: pre.deposit_receipts_start_index, }); *pre_state = post; diff --git a/consensus/state_processing/src/upgrade/eip6110.rs b/consensus/state_processing/src/upgrade/eip6110.rs new file mode 100644 index 00000000000..0afe5f0e118 --- /dev/null +++ b/consensus/state_processing/src/upgrade/eip6110.rs @@ -0,0 +1,72 @@ +use std::mem; +use types::{BeaconState, BeaconStateEip6110, BeaconStateError as Error, ChainSpec, EthSpec, Fork}; + +/// Transform a `Eip4844` state into an `Eip6110` state. +pub fn upgrade_to_eip6110( + pre_state: &mut BeaconState, + spec: &ChainSpec, +) -> Result<(), Error> { + let epoch = pre_state.current_epoch(); + let pre = pre_state.as_eip4844_mut()?; + + let previous_fork_version = pre.fork.current_version; + + let post = BeaconState::Eip6110(BeaconStateEip6110 { + // Versioning + genesis_time: pre.genesis_time, + genesis_validators_root: pre.genesis_validators_root, + slot: pre.slot, + fork: Fork { + previous_version: previous_fork_version, + current_version: spec.eip6110_fork_version, + epoch, + }, + // History + latest_block_header: pre.latest_block_header.clone(), + block_roots: pre.block_roots.clone(), + state_roots: pre.state_roots.clone(), + historical_roots: mem::take(&mut pre.historical_roots), + // Eth1 + eth1_data: pre.eth1_data.clone(), + eth1_data_votes: mem::take(&mut pre.eth1_data_votes), + eth1_deposit_index: pre.eth1_deposit_index, + // Registry + validators: mem::take(&mut pre.validators), + balances: mem::take(&mut pre.balances), + // Randomness + randao_mixes: pre.randao_mixes.clone(), + // Slashings + slashings: pre.slashings.clone(), + // Participation + previous_epoch_participation: mem::take(&mut pre.previous_epoch_participation), + current_epoch_participation: mem::take(&mut pre.current_epoch_participation), + // Finality + justification_bits: pre.justification_bits.clone(), + previous_justified_checkpoint: pre.previous_justified_checkpoint, + current_justified_checkpoint: pre.current_justified_checkpoint, + finalized_checkpoint: pre.finalized_checkpoint, + // Inactivity + inactivity_scores: mem::take(&mut pre.inactivity_scores), + // Sync committees + current_sync_committee: pre.current_sync_committee.clone(), + next_sync_committee: pre.next_sync_committee.clone(), + // Execution + latest_execution_payload_header: pre.latest_execution_payload_header.upgrade_to_eip6110(), + // Capella + next_withdrawal_index: pre.next_withdrawal_index, + next_withdrawal_validator_index: pre.next_withdrawal_validator_index, + historical_summaries: pre.historical_summaries.clone(), + // Caches + total_active_balance: pre.total_active_balance, + committee_caches: mem::take(&mut pre.committee_caches), + pubkey_cache: mem::take(&mut pre.pubkey_cache), + exit_cache: mem::take(&mut pre.exit_cache), + tree_hash_cache: mem::take(&mut pre.tree_hash_cache), + // Eip6110 + deposit_receipts_start_index: 0, + }); + + *pre_state = post; + + Ok(()) +} diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 0f26cd0e5e7..7d2a1f19888 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -1,6 +1,6 @@ use crate::beacon_block_body::{ - BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyEip4844, BeaconBlockBodyMerge, - BeaconBlockBodyRef, BeaconBlockBodyRefMut, + BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyEip4844, BeaconBlockBodyEip6110, + BeaconBlockBodyMerge, BeaconBlockBodyRef, BeaconBlockBodyRefMut, }; use crate::test_utils::TestRandom; use crate::*; @@ -17,7 +17,7 @@ use tree_hash_derive::TreeHash; /// A block of the `BeaconChain`. #[superstruct( - variants(Base, Altair, Merge, Capella, Eip4844), + variants(Base, Altair, Merge, Capella, Eip4844, Eip6110), variant_attributes( derive( Debug, @@ -74,6 +74,8 @@ pub struct BeaconBlock = FullPayload pub body: BeaconBlockBodyCapella, #[superstruct(only(Eip4844), partial_getter(rename = "body_eip4844"))] pub body: BeaconBlockBodyEip4844, + #[superstruct(only(Eip6110), partial_getter(rename = "body_eip6110"))] + pub body: BeaconBlockBodyEip6110, } pub type BlindedBeaconBlock = BeaconBlock>; @@ -132,6 +134,7 @@ impl> BeaconBlock { .or_else(|_| BeaconBlockMerge::from_ssz_bytes(bytes).map(BeaconBlock::Merge)) .or_else(|_| BeaconBlockAltair::from_ssz_bytes(bytes).map(BeaconBlock::Altair)) .or_else(|_| BeaconBlockBase::from_ssz_bytes(bytes).map(BeaconBlock::Base)) + .or_else(|_| BeaconBlockEip6110::from_ssz_bytes(bytes).map(BeaconBlock::Eip6110)) } /// Convenience accessor for the `body` as a `BeaconBlockBodyRef`. @@ -207,6 +210,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockRef<'a, T, Payl BeaconBlockRef::Merge { .. } => ForkName::Merge, BeaconBlockRef::Capella { .. } => ForkName::Capella, BeaconBlockRef::Eip4844 { .. } => ForkName::Eip4844, + BeaconBlockRef::Eip6110 { .. } => ForkName::Eip6110, }; if fork_at_slot == object_fork { @@ -560,6 +564,36 @@ impl> EmptyBlock for BeaconBlockCape } } +impl> EmptyBlock for BeaconBlockEip6110 { + /// Returns an empty Eip6110 block to be used during genesis. + fn empty(spec: &ChainSpec) -> Self { + BeaconBlockEip6110 { + slot: spec.genesis_slot, + proposer_index: 0, + parent_root: Hash256::zero(), + state_root: Hash256::zero(), + body: BeaconBlockBodyEip6110 { + randao_reveal: Signature::empty(), + eth1_data: Eth1Data { + deposit_root: Hash256::zero(), + block_hash: Hash256::zero(), + deposit_count: 0, + }, + graffiti: Graffiti::default(), + proposer_slashings: VariableList::empty(), + attester_slashings: VariableList::empty(), + attestations: VariableList::empty(), + deposits: VariableList::empty(), + voluntary_exits: VariableList::empty(), + sync_aggregate: SyncAggregate::empty(), + execution_payload: Payload::Eip6110::default(), + bls_to_execution_changes: VariableList::empty(), + blob_kzg_commitments: VariableList::empty(), + }, + } + } +} + impl> EmptyBlock for BeaconBlockEip4844 { /// Returns an empty Eip4844 block to be used during genesis. fn empty(spec: &ChainSpec) -> Self { @@ -670,6 +704,7 @@ impl_from!(BeaconBlockAltair, >, >, |body impl_from!(BeaconBlockMerge, >, >, |body: BeaconBlockBodyMerge<_, _>| body.into()); impl_from!(BeaconBlockCapella, >, >, |body: BeaconBlockBodyCapella<_, _>| body.into()); impl_from!(BeaconBlockEip4844, >, >, |body: BeaconBlockBodyEip4844<_, _>| body.into()); +impl_from!(BeaconBlockEip6110, >, >, |body: BeaconBlockBodyEip6110<_, _>| body.into()); // We can clone blocks with payloads to blocks without payloads, without cloning the payload. macro_rules! impl_clone_as_blinded { @@ -702,6 +737,7 @@ impl_clone_as_blinded!(BeaconBlockAltair, >, >, >); impl_clone_as_blinded!(BeaconBlockCapella, >, >); impl_clone_as_blinded!(BeaconBlockEip4844, >, >); +impl_clone_as_blinded!(BeaconBlockEip6110, >, >); // A reference to a full beacon block can be cloned into a blinded beacon block, without cloning the // execution payload. @@ -853,10 +889,13 @@ mod tests { let capella_slot = capella_epoch.start_slot(E::slots_per_epoch()); let eip4844_epoch = capella_epoch + 1; let eip4844_slot = eip4844_epoch.start_slot(E::slots_per_epoch()); + let eip6110_epoch = eip4844_epoch + 1; + let eip6110_slot = eip6110_epoch.start_slot(E::slots_per_epoch()); spec.altair_fork_epoch = Some(altair_epoch); spec.capella_fork_epoch = Some(capella_epoch); spec.eip4844_fork_epoch = Some(eip4844_epoch); + spec.eip6110_fork_epoch = Some(eip6110_epoch); // BeaconBlockBase { @@ -945,5 +984,27 @@ mod tests { BeaconBlock::from_ssz_bytes(&bad_block.as_ssz_bytes(), &spec) .expect_err("bad eip4844 block cannot be decoded"); } + + // BeaconBlockEip6110 + { + let good_block = BeaconBlock::Eip6110(BeaconBlockEip6110 { + slot: eip6110_slot, + ..<_>::random_for_test(rng) + }); + // It's invalid to have an Eip4844 block with a epoch lower than the fork epoch. + let bad_block = { + let mut bad = good_block.clone(); + *bad.slot_mut() = eip4844_slot; + bad + }; + + assert_eq!( + BeaconBlock::from_ssz_bytes(&good_block.as_ssz_bytes(), &spec) + .expect("good eip6110 block can be decoded"), + good_block + ); + BeaconBlock::from_ssz_bytes(&bad_block.as_ssz_bytes(), &spec) + .expect_err("bad eip6110 block cannot be decoded"); + } } } diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index c7173965224..0506763a925 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -1,3 +1,4 @@ +use crate::payload::{BlindedPayloadEip6110, FullPayloadEip6110}; use crate::*; use crate::{blobs_sidecar::KzgCommitments, test_utils::TestRandom}; use derivative::Derivative; @@ -13,7 +14,7 @@ use tree_hash_derive::TreeHash; /// /// This *superstruct* abstracts over the hard-fork. #[superstruct( - variants(Base, Altair, Merge, Capella, Eip4844), + variants(Base, Altair, Merge, Capella, Eip4844, Eip6110), variant_attributes( derive( Debug, @@ -51,7 +52,7 @@ pub struct BeaconBlockBody = FullPay pub attestations: VariableList, T::MaxAttestations>, pub deposits: VariableList, pub voluntary_exits: VariableList, - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub sync_aggregate: SyncAggregate, // We flatten the execution payload so that serde can use the name of the inner type, // either `execution_payload` for full payloads, or `execution_payload_header` for blinded @@ -65,10 +66,13 @@ pub struct BeaconBlockBody = FullPay #[superstruct(only(Eip4844), partial_getter(rename = "execution_payload_eip4844"))] #[serde(flatten)] pub execution_payload: Payload::Eip4844, - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Eip6110), partial_getter(rename = "execution_payload_eip6110"))] + #[serde(flatten)] + pub execution_payload: Payload::Eip6110, + #[superstruct(only(Capella, Eip4844, Eip6110))] pub bls_to_execution_changes: VariableList, - #[superstruct(only(Eip4844))] + #[superstruct(only(Eip4844, Eip6110))] pub blob_kzg_commitments: KzgCommitments, #[superstruct(only(Base, Altair))] #[ssz(skip_serializing, skip_deserializing)] @@ -91,6 +95,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, T, Self::Merge(body) => Ok(Payload::Ref::from(&body.execution_payload)), Self::Capella(body) => Ok(Payload::Ref::from(&body.execution_payload)), Self::Eip4844(body) => Ok(Payload::Ref::from(&body.execution_payload)), + Self::Eip6110(body) => Ok(Payload::Ref::from(&body.execution_payload)), } } } @@ -104,6 +109,7 @@ impl<'a, T: EthSpec> BeaconBlockBodyRef<'a, T> { BeaconBlockBodyRef::Merge { .. } => ForkName::Merge, BeaconBlockBodyRef::Capella { .. } => ForkName::Capella, BeaconBlockBodyRef::Eip4844 { .. } => ForkName::Eip4844, + BeaconBlockBodyRef::Eip6110 { .. } => ForkName::Eip6110, } } } @@ -372,6 +378,50 @@ impl From>> } } +impl From>> + for ( + BeaconBlockBodyEip6110>, + Option>, + ) +{ + fn from(body: BeaconBlockBodyEip6110>) -> Self { + let BeaconBlockBodyEip6110 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: FullPayloadEip6110 { execution_payload }, + bls_to_execution_changes, + blob_kzg_commitments, + } = body; + + ( + BeaconBlockBodyEip6110 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: BlindedPayloadEip6110 { + execution_payload_header: From::from(&execution_payload), + }, + bls_to_execution_changes, + blob_kzg_commitments, + }, + Some(execution_payload), + ) + } +} + // We can clone a full block into a blinded block, without cloning the payload. impl BeaconBlockBodyBase> { pub fn clone_as_blinded(&self) -> BeaconBlockBodyBase> { @@ -489,6 +539,42 @@ impl BeaconBlockBodyEip4844> { } } +impl BeaconBlockBodyEip6110> { + pub fn clone_as_blinded(&self) -> BeaconBlockBodyEip6110> { + let BeaconBlockBodyEip6110 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: FullPayloadEip6110 { execution_payload }, + bls_to_execution_changes, + blob_kzg_commitments, + } = self; + + BeaconBlockBodyEip6110 { + randao_reveal: randao_reveal.clone(), + eth1_data: eth1_data.clone(), + graffiti: *graffiti, + proposer_slashings: proposer_slashings.clone(), + attester_slashings: attester_slashings.clone(), + attestations: attestations.clone(), + deposits: deposits.clone(), + voluntary_exits: voluntary_exits.clone(), + sync_aggregate: sync_aggregate.clone(), + execution_payload: BlindedPayloadEip6110 { + execution_payload_header: execution_payload.into(), + }, + bls_to_execution_changes: bls_to_execution_changes.clone(), + blob_kzg_commitments: blob_kzg_commitments.clone(), + } + } +} + impl From>> for ( BeaconBlockBody>, diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 4f3a68e3b4f..7f7311b81e2 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -1,5 +1,6 @@ use self::committee_cache::get_active_validator_indices; use self::exit_cache::ExitCache; +use crate::execution_payload_header::ExecutionPayloadHeaderEip6110; use crate::test_utils::TestRandom; use crate::*; use compare_fields::CompareFields; @@ -176,7 +177,7 @@ impl From for Hash256 { /// The state of the `BeaconChain` at some slot. #[superstruct( - variants(Base, Altair, Merge, Capella, Eip4844), + variants(Base, Altair, Merge, Capella, Eip4844, Eip6110), variant_attributes( derive( Derivative, @@ -256,9 +257,9 @@ where pub current_epoch_attestations: VariableList, T::MaxPendingAttestations>, // Participation (Altair and later) - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub previous_epoch_participation: VariableList, - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub current_epoch_participation: VariableList, // Finality @@ -273,13 +274,13 @@ where // Inactivity #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub inactivity_scores: VariableList, // Light-client sync committees - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub current_sync_committee: Arc>, - #[superstruct(only(Altair, Merge, Capella, Eip4844))] + #[superstruct(only(Altair, Merge, Capella, Eip4844, Eip6110))] pub next_sync_committee: Arc>, // Execution @@ -298,18 +299,23 @@ where partial_getter(rename = "latest_execution_payload_header_eip4844") )] pub latest_execution_payload_header: ExecutionPayloadHeaderEip4844, + #[superstruct( + only(Eip6110), + partial_getter(rename = "latest_execution_payload_header_eip6110") + )] + pub latest_execution_payload_header: ExecutionPayloadHeaderEip6110, // Capella - #[superstruct(only(Capella, Eip4844), partial_getter(copy))] + #[superstruct(only(Capella, Eip4844, Eip6110), partial_getter(copy))] #[serde(with = "eth2_serde_utils::quoted_u64")] pub next_withdrawal_index: u64, - #[superstruct(only(Capella, Eip4844), partial_getter(copy))] + #[superstruct(only(Capella, Eip4844, Eip6110), partial_getter(copy))] #[serde(with = "eth2_serde_utils::quoted_u64")] pub next_withdrawal_validator_index: u64, // Deep history valid from Capella onwards. - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Eip4844, Eip6110))] pub historical_summaries: VariableList, - #[superstruct(only(Capella, Eip4844), partial_getter(copy))] + #[superstruct(only(Eip6110), partial_getter(copy))] #[serde(with = "eth2_serde_utils::quoted_u64")] pub deposit_receipts_start_index: u64, @@ -424,6 +430,7 @@ impl BeaconState { BeaconState::Merge { .. } => ForkName::Merge, BeaconState::Capella { .. } => ForkName::Capella, BeaconState::Eip4844 { .. } => ForkName::Eip4844, + BeaconState::Eip6110 { .. } => ForkName::Eip6110, }; if fork_at_slot == object_fork { @@ -726,6 +733,9 @@ impl BeaconState { BeaconState::Eip4844(state) => Ok(ExecutionPayloadHeaderRef::Eip4844( &state.latest_execution_payload_header, )), + BeaconState::Eip6110(state) => Ok(ExecutionPayloadHeaderRef::Eip6110( + &state.latest_execution_payload_header, + )), } } @@ -743,6 +753,9 @@ impl BeaconState { BeaconState::Eip4844(state) => Ok(ExecutionPayloadHeaderRefMut::Eip4844( &mut state.latest_execution_payload_header, )), + BeaconState::Eip6110(state) => Ok(ExecutionPayloadHeaderRefMut::Eip6110( + &mut state.latest_execution_payload_header, + )), } } @@ -1172,6 +1185,7 @@ impl BeaconState { BeaconState::Merge(state) => (&mut state.validators, &mut state.balances), BeaconState::Capella(state) => (&mut state.validators, &mut state.balances), BeaconState::Eip4844(state) => (&mut state.validators, &mut state.balances), + BeaconState::Eip6110(state) => (&mut state.validators, &mut state.balances), } } @@ -1370,6 +1384,7 @@ impl BeaconState { BeaconState::Merge(state) => Ok(&mut state.current_epoch_participation), BeaconState::Capella(state) => Ok(&mut state.current_epoch_participation), BeaconState::Eip4844(state) => Ok(&mut state.current_epoch_participation), + BeaconState::Eip6110(state) => Ok(&mut state.current_epoch_participation), } } else if epoch == self.previous_epoch() { match self { @@ -1378,6 +1393,7 @@ impl BeaconState { BeaconState::Merge(state) => Ok(&mut state.previous_epoch_participation), BeaconState::Capella(state) => Ok(&mut state.previous_epoch_participation), BeaconState::Eip4844(state) => Ok(&mut state.previous_epoch_participation), + BeaconState::Eip6110(state) => Ok(&mut state.previous_epoch_participation), } } else { Err(BeaconStateError::EpochOutOfBounds) @@ -1684,6 +1700,7 @@ impl BeaconState { BeaconState::Merge(inner) => BeaconState::Merge(inner.clone()), BeaconState::Capella(inner) => BeaconState::Capella(inner.clone()), BeaconState::Eip4844(inner) => BeaconState::Eip4844(inner.clone()), + BeaconState::Eip6110(inner) => BeaconState::Eip6110(inner.clone()), }; if config.committee_caches { *res.committee_caches_mut() = self.committee_caches().clone(); @@ -1853,6 +1870,7 @@ impl CompareFields for BeaconState { (BeaconState::Merge(x), BeaconState::Merge(y)) => x.compare_fields(y), (BeaconState::Capella(x), BeaconState::Capella(y)) => x.compare_fields(y), (BeaconState::Eip4844(x), BeaconState::Eip4844(y)) => x.compare_fields(y), + (BeaconState::Eip6110(x), BeaconState::Eip6110(y)) => x.compare_fields(y), _ => panic!("compare_fields: mismatched state variants",), } } diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 806632a6021..c7cc38648b0 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -261,15 +261,18 @@ impl ChainSpec { /// Returns the name of the fork which is active at `epoch`. pub fn fork_name_at_epoch(&self, epoch: Epoch) -> ForkName { - match self.eip4844_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Eip4844, - _ => match self.capella_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Capella, - _ => match self.bellatrix_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Merge, - _ => match self.altair_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Altair, - _ => ForkName::Base, + match self.eip6110_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Eip6110, + _ => match self.eip4844_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Eip4844, + _ => match self.capella_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Capella, + _ => match self.bellatrix_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Merge, + _ => match self.altair_fork_epoch { + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Altair, + _ => ForkName::Base, + }, }, }, }, @@ -284,6 +287,7 @@ impl ChainSpec { ForkName::Merge => self.bellatrix_fork_version, ForkName::Capella => self.capella_fork_version, ForkName::Eip4844 => self.eip4844_fork_version, + ForkName::Eip6110 => self.eip6110_fork_version, } } @@ -295,6 +299,7 @@ impl ChainSpec { ForkName::Merge => self.bellatrix_fork_epoch, ForkName::Capella => self.capella_fork_epoch, ForkName::Eip4844 => self.eip4844_fork_epoch, + ForkName::Eip6110 => self.eip6110_fork_epoch, } } @@ -306,6 +311,7 @@ impl ChainSpec { BeaconState::Merge(_) => self.inactivity_penalty_quotient_bellatrix, BeaconState::Capella(_) => self.inactivity_penalty_quotient_bellatrix, BeaconState::Eip4844(_) => self.inactivity_penalty_quotient_bellatrix, + BeaconState::Eip6110(_) => self.inactivity_penalty_quotient_bellatrix, } } @@ -320,6 +326,7 @@ impl ChainSpec { BeaconState::Merge(_) => self.proportional_slashing_multiplier_bellatrix, BeaconState::Capella(_) => self.proportional_slashing_multiplier_bellatrix, BeaconState::Eip4844(_) => self.proportional_slashing_multiplier_bellatrix, + BeaconState::Eip6110(_) => self.proportional_slashing_multiplier_bellatrix, } } @@ -334,6 +341,7 @@ impl ChainSpec { BeaconState::Merge(_) => self.min_slashing_penalty_quotient_bellatrix, BeaconState::Capella(_) => self.min_slashing_penalty_quotient_bellatrix, BeaconState::Eip4844(_) => self.min_slashing_penalty_quotient_bellatrix, + BeaconState::Eip6110(_) => self.min_slashing_penalty_quotient_bellatrix, } } @@ -724,6 +732,9 @@ impl ChainSpec { // Eip4844 eip4844_fork_version: [0x04, 0x00, 0x00, 0x01], eip4844_fork_epoch: None, + // Eip6110 + eip6110_fork_version: [0x05, 0x00, 0x00, 0x01], + eip6110_fork_epoch: None, // Other network_id: 2, // lighthouse testnet network id deposit_chain_id: 5, @@ -996,6 +1007,14 @@ pub struct Config { #[serde(deserialize_with = "deserialize_fork_epoch")] pub eip4844_fork_epoch: Option>, + #[serde(default = "default_eip6110_fork_version")] + #[serde(with = "eth2_serde_utils::bytes_4_hex")] + eip6110_fork_version: [u8; 4], + #[serde(default)] + #[serde(serialize_with = "serialize_fork_epoch")] + #[serde(deserialize_with = "deserialize_fork_epoch")] + pub eip6110_fork_epoch: Option>, + #[serde(with = "eth2_serde_utils::quoted_u64")] seconds_per_slot: u64, #[serde(with = "eth2_serde_utils::quoted_u64")] @@ -1043,6 +1062,11 @@ fn default_eip4844_fork_version() -> [u8; 4] { [0xff, 0xff, 0xff, 0xff] } +fn default_eip6110_fork_version() -> [u8; 4] { + // This value shouldn't be used. + [0xff, 0xff, 0xff, 0xff] +} + /// Placeholder value: 2^256-2^10 (115792089237316195423570985008687907853269984665640564039457584007913129638912). /// /// Taken from https://github.com/ethereum/consensus-specs/blob/d5e4828aecafaf1c57ef67a5f23c4ae7b08c5137/configs/mainnet.yaml#L15-L16 @@ -1147,6 +1171,10 @@ impl Config { eip4844_fork_epoch: spec .eip4844_fork_epoch .map(|epoch| MaybeQuoted { value: epoch }), + eip6110_fork_version: spec.eip6110_fork_version, + eip6110_fork_epoch: spec + .eip6110_fork_epoch + .map(|epoch| MaybeQuoted { value: epoch }), seconds_per_slot: spec.seconds_per_slot, seconds_per_eth1_block: spec.seconds_per_eth1_block, @@ -1196,6 +1224,8 @@ impl Config { capella_fork_version, eip4844_fork_epoch, eip4844_fork_version, + eip6110_fork_epoch, + eip6110_fork_version, seconds_per_slot, seconds_per_eth1_block, min_validator_withdrawability_delay, @@ -1230,6 +1260,8 @@ impl Config { capella_fork_version, eip4844_fork_epoch: eip4844_fork_epoch.map(|q| q.value), eip4844_fork_version, + eip6110_fork_epoch: eip6110_fork_epoch.map(|q| q.value), + eip6110_fork_version, seconds_per_slot, seconds_per_eth1_block, min_validator_withdrawability_delay, diff --git a/consensus/types/src/eip6110.rs b/consensus/types/src/eip6110.rs index 9b375b2aa08..230db75badb 100644 --- a/consensus/types/src/eip6110.rs +++ b/consensus/types/src/eip6110.rs @@ -8,7 +8,7 @@ use tree_hash_derive::TreeHash; #[derive( arbitrary::Arbitrary, Debug, - PartialEq, + // PartialEq, // Eq, Hash, Clone, @@ -29,6 +29,18 @@ pub struct DepositReceipt { pub index: u64, } +// Manually implement the Eq trait for DepositReceipt +impl Eq for DepositReceipt {} + +impl PartialEq for DepositReceipt { + fn eq(&self, other: &DepositReceipt) -> bool { + self.pubkey == other.pubkey + && self.withdrawal_credentials == other.withdrawal_credentials + && self.amount == other.amount + && self.index == other.index + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 66316a699d0..2584d03336a 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -252,6 +252,11 @@ pub trait EthSpec: Self::MaxWithdrawalsPerPayload::to_usize() } + /// Returns the `MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD` constant for this specification. + fn max_deposit_receipts_per_payload() -> usize { + Self::MaxDepositReceiptsPerPayload::to_usize() + } + /// Returns the `MAX_BLOBS_PER_BLOCK` constant for this specification. fn max_blobs_per_block() -> usize { Self::MaxBlobsPerBlock::to_usize() diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 2e02326de4f..42d3f43839f 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -18,7 +18,7 @@ pub type DepositReceipts = VariableList::MaxDepositReceiptsPerPayload>; #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Eip4844, Eip6110), variant_attributes( derive( Default, @@ -80,7 +80,7 @@ pub struct ExecutionPayload { #[serde(with = "eth2_serde_utils::quoted_u256")] #[superstruct(getter(copy))] pub base_fee_per_gas: Uint256, - #[superstruct(only(Eip4844))] + #[superstruct(only(Eip4844, Eip6110))] #[serde(with = "eth2_serde_utils::quoted_u256")] #[superstruct(getter(copy))] pub excess_data_gas: Uint256, @@ -88,9 +88,9 @@ pub struct ExecutionPayload { pub block_hash: ExecutionBlockHash, #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Eip4844, Eip6110))] pub withdrawals: Withdrawals, - #[superstruct(only(Eip4844))] + #[superstruct(only(Eip6110))] pub deposit_receipts: DepositReceipts, } @@ -113,6 +113,7 @@ impl ExecutionPayload { ForkName::Merge => ExecutionPayloadMerge::from_ssz_bytes(bytes).map(Self::Merge), ForkName::Capella => ExecutionPayloadCapella::from_ssz_bytes(bytes).map(Self::Capella), ForkName::Eip4844 => ExecutionPayloadEip4844::from_ssz_bytes(bytes).map(Self::Eip4844), + ForkName::Eip6110 => ExecutionPayloadEip6110::from_ssz_bytes(bytes).map(Self::Eip6110), } } @@ -152,6 +153,19 @@ impl ExecutionPayload { // Max size of variable length `withdrawals` field + (T::max_withdrawals_per_payload() * ::ssz_fixed_len()) } + + #[allow(clippy::integer_arithmetic)] + /// Returns the maximum size of an execution payload. + pub fn max_execution_payload_eip6110_size() -> usize { + // Fixed part + ExecutionPayloadEip6110::::default().as_ssz_bytes().len() + // Max size of variable length `extra_data` field + + (T::max_extra_data_bytes() * ::ssz_fixed_len()) + // Max size of variable length `transactions` field + + (T::max_transactions_per_payload() * (ssz::BYTES_PER_LENGTH_OFFSET + T::max_bytes_per_transaction())) + // Max size of variable length `deposit_receipts` field + + (T::max_deposit_receipts_per_payload() * ::ssz_fixed_len()) + } } impl ForkVersionDeserialize for ExecutionPayload { @@ -167,6 +181,7 @@ impl ForkVersionDeserialize for ExecutionPayload { ForkName::Merge => Self::Merge(serde_json::from_value(value).map_err(convert_err)?), ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), ForkName::Eip4844 => Self::Eip4844(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Eip6110 => Self::Eip6110(serde_json::from_value(value).map_err(convert_err)?), ForkName::Base | ForkName::Altair => { return Err(serde::de::Error::custom(format!( "ExecutionPayload failed to deserialize: unsupported fork '{}'", diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index fde2fa8880d..dd7b879fb8b 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -9,7 +9,7 @@ use tree_hash_derive::TreeHash; use BeaconStateError; #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Eip4844, Eip6110), variant_attributes( derive( Default, @@ -70,7 +70,7 @@ pub struct ExecutionPayloadHeader { #[serde(with = "eth2_serde_utils::quoted_u256")] #[superstruct(getter(copy))] pub base_fee_per_gas: Uint256, - #[superstruct(only(Eip4844))] + #[superstruct(only(Eip4844, Eip6110))] #[serde(with = "eth2_serde_utils::quoted_u256")] #[superstruct(getter(copy))] pub excess_data_gas: Uint256, @@ -78,10 +78,10 @@ pub struct ExecutionPayloadHeader { pub block_hash: ExecutionBlockHash, #[superstruct(getter(copy))] pub transactions_root: Hash256, - #[superstruct(only(Capella, Eip4844))] + #[superstruct(only(Capella, Eip4844, Eip6110))] #[superstruct(getter(copy))] pub withdrawals_root: Hash256, - #[superstruct(only(Eip4844))] + #[superstruct(only(Eip6110))] #[superstruct(getter(copy))] pub deposit_receipts_root: Hash256, } @@ -103,6 +103,9 @@ impl ExecutionPayloadHeader { ForkName::Eip4844 => { ExecutionPayloadHeaderEip4844::from_ssz_bytes(bytes).map(Self::Eip4844) } + ForkName::Eip6110 => { + ExecutionPayloadHeaderEip6110::from_ssz_bytes(bytes).map(Self::Eip6110) + } } } } @@ -158,6 +161,29 @@ impl ExecutionPayloadHeaderCapella { block_hash: self.block_hash, transactions_root: self.transactions_root, withdrawals_root: self.withdrawals_root, + } + } +} + +impl ExecutionPayloadHeaderEip4844 { + pub fn upgrade_to_eip6110(&self) -> ExecutionPayloadHeaderEip6110 { + ExecutionPayloadHeaderEip6110 { + parent_hash: self.parent_hash, + fee_recipient: self.fee_recipient, + state_root: self.state_root, + receipts_root: self.receipts_root, + logs_bloom: self.logs_bloom.clone(), + prev_randao: self.prev_randao, + block_number: self.block_number, + gas_limit: self.gas_limit, + gas_used: self.gas_used, + timestamp: self.timestamp, + extra_data: self.extra_data.clone(), + base_fee_per_gas: self.base_fee_per_gas, + excess_data_gas: Uint256::zero(), + block_hash: self.block_hash, + transactions_root: self.transactions_root, + withdrawals_root: self.withdrawals_root, deposit_receipts_root: Hash256::zero(), } } @@ -207,6 +233,29 @@ impl<'a, T: EthSpec> From<&'a ExecutionPayloadCapella> for ExecutionPayloadHe impl<'a, T: EthSpec> From<&'a ExecutionPayloadEip4844> for ExecutionPayloadHeaderEip4844 { fn from(payload: &'a ExecutionPayloadEip4844) -> Self { + Self { + parent_hash: payload.parent_hash, + fee_recipient: payload.fee_recipient, + state_root: payload.state_root, + receipts_root: payload.receipts_root, + logs_bloom: payload.logs_bloom.clone(), + prev_randao: payload.prev_randao, + block_number: payload.block_number, + gas_limit: payload.gas_limit, + gas_used: payload.gas_used, + timestamp: payload.timestamp, + extra_data: payload.extra_data.clone(), + base_fee_per_gas: payload.base_fee_per_gas, + excess_data_gas: payload.excess_data_gas, + block_hash: payload.block_hash, + transactions_root: payload.transactions.tree_hash_root(), + withdrawals_root: payload.withdrawals.tree_hash_root(), + } + } +} + +impl<'a, T: EthSpec> From<&'a ExecutionPayloadEip6110> for ExecutionPayloadHeaderEip6110 { + fn from(payload: &'a ExecutionPayloadEip6110) -> Self { Self { parent_hash: payload.parent_hash, fee_recipient: payload.fee_recipient, @@ -249,6 +298,12 @@ impl<'a, T: EthSpec> From<&'a Self> for ExecutionPayloadHeaderEip4844 { } } +impl<'a, T: EthSpec> From<&'a Self> for ExecutionPayloadHeaderEip6110 { + fn from(payload: &'a Self) -> Self { + payload.clone() + } +} + impl<'a, T: EthSpec> From> for ExecutionPayloadHeader { fn from(payload: ExecutionPayloadRef<'a, T>) -> Self { map_execution_payload_ref_into_execution_payload_header!( @@ -291,6 +346,18 @@ impl TryFrom> for ExecutionPayloadHeaderEi } } +impl TryFrom> for ExecutionPayloadHeaderEip6110 { + type Error = BeaconStateError; + fn try_from(header: ExecutionPayloadHeader) -> Result { + match header { + ExecutionPayloadHeader::Eip6110(execution_payload_header) => { + Ok(execution_payload_header) + } + _ => Err(BeaconStateError::IncorrectStateVariant), + } + } +} + impl ForkVersionDeserialize for ExecutionPayloadHeader { fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( value: serde_json::value::Value, @@ -307,6 +374,7 @@ impl ForkVersionDeserialize for ExecutionPayloadHeader { ForkName::Merge => Self::Merge(serde_json::from_value(value).map_err(convert_err)?), ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), ForkName::Eip4844 => Self::Eip4844(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Eip6110 => Self::Eip6110(serde_json::from_value(value).map_err(convert_err)?), ForkName::Base | ForkName::Altair => { return Err(serde::de::Error::custom(format!( "ExecutionPayloadHeader failed to deserialize: unsupported fork '{}'", diff --git a/consensus/types/src/fork_context.rs b/consensus/types/src/fork_context.rs index f5221dd913d..3bf8687c425 100644 --- a/consensus/types/src/fork_context.rs +++ b/consensus/types/src/fork_context.rs @@ -61,6 +61,13 @@ impl ForkContext { )); } + if spec.eip6110_fork_epoch.is_some() { + fork_to_digest.push(( + ForkName::Eip6110, + ChainSpec::compute_fork_digest(spec.eip6110_fork_version, genesis_validators_root), + )); + } + let fork_to_digest: HashMap = fork_to_digest.into_iter().collect(); let digest_to_fork = fork_to_digest diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index 89eaff7985d..6160d92b90f 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -13,6 +13,7 @@ pub enum ForkName { Merge, Capella, Eip4844, + Eip6110, } impl ForkName { @@ -23,6 +24,7 @@ impl ForkName { ForkName::Merge, ForkName::Capella, ForkName::Eip4844, + ForkName::Eip6110, ] } @@ -36,6 +38,7 @@ impl ForkName { spec.bellatrix_fork_epoch = None; spec.capella_fork_epoch = None; spec.eip4844_fork_epoch = None; + spec.eip6110_fork_epoch = None; spec } ForkName::Altair => { @@ -43,6 +46,7 @@ impl ForkName { spec.bellatrix_fork_epoch = None; spec.capella_fork_epoch = None; spec.eip4844_fork_epoch = None; + spec.eip6110_fork_epoch = None; spec } ForkName::Merge => { @@ -50,6 +54,7 @@ impl ForkName { spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec.capella_fork_epoch = None; spec.eip4844_fork_epoch = None; + spec.eip6110_fork_epoch = None; spec } ForkName::Capella => { @@ -57,6 +62,7 @@ impl ForkName { spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec.capella_fork_epoch = Some(Epoch::new(0)); spec.eip4844_fork_epoch = None; + spec.eip6110_fork_epoch = None; spec } ForkName::Eip4844 => { @@ -64,6 +70,15 @@ impl ForkName { spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec.capella_fork_epoch = Some(Epoch::new(0)); spec.eip4844_fork_epoch = Some(Epoch::new(0)); + spec.eip6110_fork_epoch = None; + spec + } + ForkName::Eip6110 => { + spec.altair_fork_epoch = Some(Epoch::new(0)); + spec.bellatrix_fork_epoch = Some(Epoch::new(0)); + spec.capella_fork_epoch = Some(Epoch::new(0)); + spec.eip4844_fork_epoch = Some(Epoch::new(0)); + spec.eip6110_fork_epoch = Some(Epoch::new(0)); spec } } @@ -79,6 +94,7 @@ impl ForkName { ForkName::Merge => Some(ForkName::Altair), ForkName::Capella => Some(ForkName::Merge), ForkName::Eip4844 => Some(ForkName::Capella), + ForkName::Eip6110 => Some(ForkName::Eip4844), } } @@ -91,7 +107,8 @@ impl ForkName { ForkName::Altair => Some(ForkName::Merge), ForkName::Merge => Some(ForkName::Capella), ForkName::Capella => Some(ForkName::Eip4844), - ForkName::Eip4844 => None, + ForkName::Eip4844 => Some(ForkName::Eip6110), + ForkName::Eip6110 => None, } } } @@ -141,6 +158,10 @@ macro_rules! map_fork_name_with { let (value, extra_data) = $body; ($t::Eip4844(value), extra_data) } + ForkName::Eip6110 => { + let (value, extra_data) = $body; + ($t::Eip6110(value), extra_data) + } } }; } @@ -155,6 +176,7 @@ impl FromStr for ForkName { "bellatrix" | "merge" => ForkName::Merge, "capella" => ForkName::Capella, "eip4844" => ForkName::Eip4844, + "eip6110" => ForkName::Eip6110, _ => return Err(format!("unknown fork name: {}", fork_name)), }) } @@ -168,6 +190,7 @@ impl Display for ForkName { ForkName::Merge => "bellatrix".fmt(f), ForkName::Capella => "capella".fmt(f), ForkName::Eip4844 => "eip4844".fmt(f), + ForkName::Eip6110 => "eip6110".fmt(f), } } } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index b38a6080b6c..04514aa5617 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -111,11 +111,13 @@ pub use crate::attestation_duty::AttestationDuty; pub use crate::attester_slashing::AttesterSlashing; pub use crate::beacon_block::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockEip4844, - BeaconBlockMerge, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, EmptyBlock, + BeaconBlockEip6110, BeaconBlockMerge, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, + EmptyBlock, }; pub use crate::beacon_block_body::{ BeaconBlockBody, BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyCapella, - BeaconBlockBodyEip4844, BeaconBlockBodyMerge, BeaconBlockBodyRef, BeaconBlockBodyRefMut, + BeaconBlockBodyEip4844, BeaconBlockBodyEip6110, BeaconBlockBodyMerge, BeaconBlockBodyRef, + BeaconBlockBodyRefMut, }; pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; @@ -140,11 +142,13 @@ pub use crate::execution_block_hash::ExecutionBlockHash; pub use crate::execution_block_header::ExecutionBlockHeader; pub use crate::execution_payload::{ DepositReceipts, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadEip4844, - ExecutionPayloadMerge, ExecutionPayloadRef, Transaction, Transactions, Withdrawals, + ExecutionPayloadEip6110, ExecutionPayloadMerge, ExecutionPayloadRef, Transaction, Transactions, + Withdrawals, }; pub use crate::execution_payload_header::{ ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderEip4844, - ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRef, ExecutionPayloadHeaderRefMut, + ExecutionPayloadHeaderEip6110, ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRef, + ExecutionPayloadHeaderRefMut, }; pub use crate::fork::Fork; pub use crate::fork_context::ForkContext; @@ -162,8 +166,9 @@ pub use crate::participation_flags::ParticipationFlags; pub use crate::participation_list::ParticipationList; pub use crate::payload::{ AbstractExecPayload, BlindedPayload, BlindedPayloadCapella, BlindedPayloadEip4844, - BlindedPayloadMerge, BlindedPayloadRef, BlockType, ExecPayload, FullPayload, - FullPayloadCapella, FullPayloadEip4844, FullPayloadMerge, FullPayloadRef, OwnedExecPayload, + BlindedPayloadEip6110, BlindedPayloadMerge, BlindedPayloadRef, BlockType, ExecPayload, + FullPayload, FullPayloadCapella, FullPayloadEip4844, FullPayloadEip6110, FullPayloadMerge, + FullPayloadRef, OwnedExecPayload, }; pub use crate::pending_attestation::PendingAttestation; pub use crate::preset::{AltairPreset, BasePreset, BellatrixPreset, CapellaPreset}; @@ -175,8 +180,8 @@ pub use crate::shuffling_id::AttestationShufflingId; pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof; pub use crate::signed_beacon_block::{ SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, - SignedBeaconBlockEip4844, SignedBeaconBlockHash, SignedBeaconBlockMerge, - SignedBlindedBeaconBlock, + SignedBeaconBlockEip4844, SignedBeaconBlockEip6110, SignedBeaconBlockHash, + SignedBeaconBlockMerge, SignedBlindedBeaconBlock, }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_block_and_blobs::SignedBeaconBlockAndBlobsSidecar; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index bbc6423c057..eeece8f96bd 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -83,12 +83,14 @@ pub trait AbstractExecPayload: + TryInto + TryInto + TryInto + + TryInto { type Ref<'a>: ExecPayload + Copy + From<&'a Self::Merge> + From<&'a Self::Capella> - + From<&'a Self::Eip4844>; + + From<&'a Self::Eip4844> + + From<&'a Self::Eip6110>; type Merge: OwnedExecPayload + Into @@ -102,12 +104,16 @@ pub trait AbstractExecPayload: + Into + for<'a> From>> + TryFrom>; + type Eip6110: OwnedExecPayload + + Into + + for<'a> From>> + + TryFrom>; fn default_at_fork(fork_name: ForkName) -> Result; } #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Eip4844, Eip6110), variant_attributes( derive( Debug, @@ -148,6 +154,8 @@ pub struct FullPayload { pub execution_payload: ExecutionPayloadCapella, #[superstruct(only(Eip4844), partial_getter(rename = "execution_payload_eip4844"))] pub execution_payload: ExecutionPayloadEip4844, + #[superstruct(only(Eip6110), partial_getter(rename = "execution_payload_eip6110"))] + pub execution_payload: ExecutionPayloadEip6110, } impl From> for ExecutionPayload { @@ -254,6 +262,9 @@ impl ExecPayload for FullPayload { FullPayload::Eip4844(ref inner) => { Ok(inner.execution_payload.withdrawals.tree_hash_root()) } + FullPayload::Eip6110(ref inner) => { + Ok(inner.execution_payload.withdrawals.tree_hash_root()) + } } } @@ -279,7 +290,11 @@ impl ExecPayload for FullPayload { // Return an "empty" or "default" value for the Capella variant Ok(Vec::new().into()) } - FullPayload::Eip4844(ref inner) => Ok(inner.execution_payload.deposit_receipts.clone()), + FullPayload::Eip4844(_) => { + // Return an "empty" or "default" value for the EIP-4844 variant + Ok(Vec::new().into()) + } + FullPayload::Eip6110(ref inner) => Ok(inner.execution_payload.deposit_receipts.clone()), } } } @@ -377,6 +392,9 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { FullPayloadRef::Eip4844(inner) => { Ok(inner.execution_payload.withdrawals.tree_hash_root()) } + FullPayloadRef::Eip6110(inner) => { + Ok(inner.execution_payload.withdrawals.tree_hash_root()) + } } } @@ -402,7 +420,11 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { // Return an "empty" or "default" value for the Capella variant Ok(Vec::new().into()) } - FullPayloadRef::Eip4844(ref inner) => { + FullPayloadRef::Eip4844(_) => { + // Return an "empty" or "default" value for the EIP-4844 variant + Ok(Vec::new().into()) + } + FullPayloadRef::Eip6110(ref inner) => { Ok(inner.execution_payload.deposit_receipts.clone()) } } @@ -414,6 +436,7 @@ impl AbstractExecPayload for FullPayload { type Merge = FullPayloadMerge; type Capella = FullPayloadCapella; type Eip4844 = FullPayloadEip4844; + type Eip6110 = FullPayloadEip6110; fn default_at_fork(fork_name: ForkName) -> Result { match fork_name { @@ -421,6 +444,7 @@ impl AbstractExecPayload for FullPayload { ForkName::Merge => Ok(FullPayloadMerge::default().into()), ForkName::Capella => Ok(FullPayloadCapella::default().into()), ForkName::Eip4844 => Ok(FullPayloadEip4844::default().into()), + ForkName::Eip6110 => Ok(FullPayloadEip6110::default().into()), } } } @@ -441,7 +465,7 @@ impl TryFrom> for FullPayload { } #[superstruct( - variants(Merge, Capella, Eip4844), + variants(Merge, Capella, Eip4844, Eip6110), variant_attributes( derive( Debug, @@ -481,6 +505,8 @@ pub struct BlindedPayload { pub execution_payload_header: ExecutionPayloadHeaderCapella, #[superstruct(only(Eip4844), partial_getter(rename = "execution_payload_eip4844"))] pub execution_payload_header: ExecutionPayloadHeaderEip4844, + #[superstruct(only(Eip6110), partial_getter(rename = "execution_payload_eip6110"))] + pub execution_payload_header: ExecutionPayloadHeaderEip6110, } impl<'a, T: EthSpec> From> for BlindedPayload { @@ -565,6 +591,9 @@ impl ExecPayload for BlindedPayload { BlindedPayload::Eip4844(ref inner) => { Ok(inner.execution_payload_header.withdrawals_root) } + BlindedPayload::Eip6110(ref inner) => { + Ok(inner.execution_payload_header.withdrawals_root) + } } } @@ -659,6 +688,9 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { BlindedPayloadRef::Eip4844(inner) => { Ok(inner.execution_payload_header.withdrawals_root) } + BlindedPayloadRef::Eip6110(inner) => { + Ok(inner.execution_payload_header.withdrawals_root) + } } } @@ -690,6 +722,10 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { // Return an "empty" or "default" value for the Eip4844 variant Ok(Vec::new().into()) } + BlindedPayloadRef::Eip6110(_) => { + // Return an "empty" or "default" value for the Eip6110 variant + Ok(Vec::new().into()) + } } } } @@ -950,12 +986,20 @@ impl_exec_payload_for_fork!( ExecutionPayloadEip4844, Eip4844 ); +impl_exec_payload_for_fork!( + BlindedPayloadEip6110, + FullPayloadEip6110, + ExecutionPayloadHeaderEip6110, + ExecutionPayloadEip6110, + Eip6110 +); impl AbstractExecPayload for BlindedPayload { type Ref<'a> = BlindedPayloadRef<'a, T>; type Merge = BlindedPayloadMerge; type Capella = BlindedPayloadCapella; type Eip4844 = BlindedPayloadEip4844; + type Eip6110 = BlindedPayloadEip6110; fn default_at_fork(fork_name: ForkName) -> Result { match fork_name { @@ -963,6 +1007,7 @@ impl AbstractExecPayload for BlindedPayload { ForkName::Merge => Ok(BlindedPayloadMerge::default().into()), ForkName::Capella => Ok(BlindedPayloadCapella::default().into()), ForkName::Eip4844 => Ok(BlindedPayloadEip4844::default().into()), + ForkName::Eip6110 => Ok(BlindedPayloadEip6110::default().into()), } } } @@ -996,6 +1041,11 @@ impl From> for BlindedPayload { execution_payload_header, }) } + ExecutionPayloadHeader::Eip6110(execution_payload_header) => { + Self::Eip6110(BlindedPayloadEip6110 { + execution_payload_header, + }) + } } } } @@ -1012,6 +1062,9 @@ impl From> for ExecutionPayloadHeader { BlindedPayload::Eip4844(blinded_payload) => { ExecutionPayloadHeader::Eip4844(blinded_payload.execution_payload_header) } + BlindedPayload::Eip6110(blinded_payload) => { + ExecutionPayloadHeader::Eip6110(blinded_payload.execution_payload_header) + } } } } diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index ae59690bf2d..1546bc1e04a 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -45,7 +45,7 @@ pub enum BlobReconstructionError { /// A `BeaconBlock` and a signature from its proposer. #[superstruct( - variants(Base, Altair, Merge, Capella, Eip4844), + variants(Base, Altair, Merge, Capella, Eip4844, Eip6110), variant_attributes( derive( Debug, @@ -86,6 +86,8 @@ pub struct SignedBeaconBlock = FullP pub message: BeaconBlockCapella, #[superstruct(only(Eip4844), partial_getter(rename = "message_eip4844"))] pub message: BeaconBlockEip4844, + #[superstruct(only(Eip6110), partial_getter(rename = "message_eip6110"))] + pub message: BeaconBlockEip6110, pub signature: Signature, } @@ -149,6 +151,9 @@ impl> SignedBeaconBlock BeaconBlock::Eip4844(message) => { SignedBeaconBlock::Eip4844(SignedBeaconBlockEip4844 { message, signature }) } + BeaconBlock::Eip6110(message) => { + SignedBeaconBlock::Eip6110(SignedBeaconBlockEip6110 { message, signature }) + } } } @@ -464,6 +469,62 @@ impl SignedBeaconBlockEip4844> { } } +impl SignedBeaconBlockEip6110> { + pub fn into_full_block( + self, + execution_payload: ExecutionPayloadEip6110, + ) -> SignedBeaconBlockEip6110> { + let SignedBeaconBlockEip6110 { + message: + BeaconBlockEip6110 { + slot, + proposer_index, + parent_root, + state_root, + body: + BeaconBlockBodyEip6110 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: BlindedPayloadEip6110 { .. }, + bls_to_execution_changes, + blob_kzg_commitments, + }, + }, + signature, + } = self; + SignedBeaconBlockEip6110 { + message: BeaconBlockEip6110 { + slot, + proposer_index, + parent_root, + state_root, + body: BeaconBlockBodyEip6110 { + randao_reveal, + eth1_data, + graffiti, + proposer_slashings, + attester_slashings, + attestations, + deposits, + voluntary_exits, + sync_aggregate, + execution_payload: FullPayloadEip6110 { execution_payload }, + bls_to_execution_changes, + blob_kzg_commitments, + }, + }, + signature, + } + } +} + impl SignedBeaconBlock> { pub fn try_into_full_block( self, @@ -481,11 +542,15 @@ impl SignedBeaconBlock> { (SignedBeaconBlock::Eip4844(block), Some(ExecutionPayload::Eip4844(payload))) => { SignedBeaconBlock::Eip4844(block.into_full_block(payload)) } + (SignedBeaconBlock::Eip6110(block), Some(ExecutionPayload::Eip6110(payload))) => { + SignedBeaconBlock::Eip6110(block.into_full_block(payload)) + } // avoid wildcard matching forks so that compiler will // direct us here when a new fork has been added (SignedBeaconBlock::Merge(_), _) => return None, (SignedBeaconBlock::Capella(_), _) => return None, (SignedBeaconBlock::Eip4844(_), _) => return None, + (SignedBeaconBlock::Eip6110(_), _) => return None, }; Some(full_block) } diff --git a/lcli/src/create_payload_header.rs b/lcli/src/create_payload_header.rs index 7700f23d9dd..6a75ec11640 100644 --- a/lcli/src/create_payload_header.rs +++ b/lcli/src/create_payload_header.rs @@ -6,7 +6,7 @@ use std::io::Write; use std::time::{SystemTime, UNIX_EPOCH}; use types::{ EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderEip4844, - ExecutionPayloadHeaderMerge, ForkName, + ExecutionPayloadHeaderEip6110, ExecutionPayloadHeaderMerge, ForkName, }; pub fn run(matches: &ArgMatches) -> Result<(), String> { @@ -48,6 +48,14 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { prev_randao: eth1_block_hash.into_root(), ..ExecutionPayloadHeaderEip4844::default() }), + ForkName::Eip6110 => ExecutionPayloadHeader::Eip6110(ExecutionPayloadHeaderEip6110 { + gas_limit, + base_fee_per_gas, + timestamp: genesis_time, + block_hash: eth1_block_hash, + prev_randao: eth1_block_hash.into_root(), + ..ExecutionPayloadHeaderEip6110::default() + }), }; let mut file = File::create(file_name).map_err(|_| "Unable to create file".to_string())?; diff --git a/lcli/src/main.rs b/lcli/src/main.rs index 92ecc78ed16..b31f1d6dd21 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -425,7 +425,7 @@ fn main() { .takes_value(true) .default_value("bellatrix") .help("The fork for which the execution payload header should be created.") - .possible_values(&["merge", "bellatrix", "capella", "eip4844"]) + .possible_values(&["merge", "bellatrix", "capella", "eip4844", "eip6110"]) ) ) .subcommand( @@ -594,6 +594,15 @@ fn main() { "The epoch at which to enable the eip4844 hard fork", ), ) + .arg( + Arg::with_name("eip6110-fork-epoch") + .long("eip6110-fork-epoch") + .value_name("EPOCH") + .takes_value(true) + .help( + "The epoch at which to enable the eip6110 hard fork", + ), + ) .arg( Arg::with_name("ttd") .long("ttd") diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index e28197e6178..43b1cd7b096 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -16,8 +16,8 @@ use types::ExecutionBlockHash; use types::{ test_utils::generate_deterministic_keypairs, Address, BeaconState, ChainSpec, Config, Epoch, Eth1Data, EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, - ExecutionPayloadHeaderEip4844, ExecutionPayloadHeaderMerge, ForkName, Hash256, Keypair, - PublicKey, Validator, + ExecutionPayloadHeaderEip4844, ExecutionPayloadHeaderEip6110, ExecutionPayloadHeaderMerge, + ForkName, Hash256, Keypair, PublicKey, Validator, }; pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Result<(), String> { @@ -86,6 +86,10 @@ pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul spec.eip4844_fork_epoch = Some(fork_epoch); } + if let Some(fork_epoch) = parse_optional(matches, "eip6110-fork-epoch")? { + spec.eip6110_fork_epoch = Some(fork_epoch); + } + if let Some(ttd) = parse_optional(matches, "ttd")? { spec.terminal_total_difficulty = ttd; } @@ -116,6 +120,10 @@ pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul ExecutionPayloadHeaderEip4844::::from_ssz_bytes(bytes.as_slice()) .map(ExecutionPayloadHeader::Eip4844) } + ForkName::Eip6110 => { + ExecutionPayloadHeaderEip6110::::from_ssz_bytes(bytes.as_slice()) + .map(ExecutionPayloadHeader::Eip6110) + } } .map_err(|e| format!("SSZ decode failed: {:?}", e)) }) diff --git a/lcli/src/parse_ssz.rs b/lcli/src/parse_ssz.rs index 0e87e330b13..7876d7e60e5 100644 --- a/lcli/src/parse_ssz.rs +++ b/lcli/src/parse_ssz.rs @@ -53,16 +53,19 @@ pub fn run_parse_ssz(matches: &ArgMatches) -> Result<(), String> { "signed_block_merge" => decode_and_print::>(&bytes, format)?, "signed_block_capella" => decode_and_print::>(&bytes, format)?, "signed_block_eip4844" => decode_and_print::>(&bytes, format)?, + "signed_block_eip6110" => decode_and_print::>(&bytes, format)?, "block_base" => decode_and_print::>(&bytes, format)?, "block_altair" => decode_and_print::>(&bytes, format)?, "block_merge" => decode_and_print::>(&bytes, format)?, "block_capella" => decode_and_print::>(&bytes, format)?, "block_eip4844" => decode_and_print::>(&bytes, format)?, + "block_eip6110" => decode_and_print::>(&bytes, format)?, "state_base" => decode_and_print::>(&bytes, format)?, "state_altair" => decode_and_print::>(&bytes, format)?, "state_merge" => decode_and_print::>(&bytes, format)?, "state_capella" => decode_and_print::>(&bytes, format)?, "state_eip4844" => decode_and_print::>(&bytes, format)?, + "state_eip6110" => decode_and_print::>(&bytes, format)?, "blobs_sidecar" => decode_and_print::>(&bytes, format)?, other => return Err(format!("Unknown type: {}", other)), }; diff --git a/scripts/local_testnet/setup.sh b/scripts/local_testnet/setup.sh index 1dc7566f6b2..05f2288658f 100755 --- a/scripts/local_testnet/setup.sh +++ b/scripts/local_testnet/setup.sh @@ -30,6 +30,7 @@ lcli \ --bellatrix-fork-epoch $BELLATRIX_FORK_EPOCH \ --capella-fork-epoch $CAPELLA_FORK_EPOCH \ --eip4844-fork-epoch $EIP4844_FORK_EPOCH \ + --eip6110-fork-epoch $EIP6110_FORK_EPOCH \ --ttd $TTD \ --eth1-block-hash $ETH1_BLOCK_HASH \ --eth1-id $CHAIN_ID \ @@ -55,6 +56,7 @@ echo Validators generated with keystore passwords at $DATADIR. GENESIS_TIME=$(lcli pretty-ssz state_merge ~/.lighthouse/local-testnet/testnet/genesis.ssz | jq | grep -Po 'genesis_time": "\K.*\d') CAPELLA_TIME=$((GENESIS_TIME + (CAPELLA_FORK_EPOCH * 32 * SECONDS_PER_SLOT))) EIP4844_TIME=$((GENESIS_TIME + (EIP4844_FORK_EPOCH * 32 * SECONDS_PER_SLOT))) +EIP6110_TIME=$((GENESIS_TIME + (EIP6110_FORK_EPOCH * 32 * SECONDS_PER_SLOT))) sed -i 's/"shanghaiTime".*$/"shanghaiTime": '"$CAPELLA_TIME"',/g' genesis.json sed -i 's/"shardingForkTime".*$/"shardingForkTime": '"$EIP4844_TIME"',/g' genesis.json diff --git a/scripts/local_testnet/vars.env b/scripts/local_testnet/vars.env index dcdf671e3f1..ba860365753 100644 --- a/scripts/local_testnet/vars.env +++ b/scripts/local_testnet/vars.env @@ -42,6 +42,7 @@ ALTAIR_FORK_EPOCH=0 BELLATRIX_FORK_EPOCH=0 CAPELLA_FORK_EPOCH=1 EIP4844_FORK_EPOCH=2 +EIP6110_FORK_EPOCH=3 TTD=0 diff --git a/scripts/tests/vars.env b/scripts/tests/vars.env index e7a688769a9..1c39640c026 100644 --- a/scripts/tests/vars.env +++ b/scripts/tests/vars.env @@ -37,6 +37,7 @@ ALTAIR_FORK_EPOCH=18446744073709551615 BELLATRIX_FORK_EPOCH=18446744073709551615 CAPELLA_FORK_EPOCH=18446744073709551615 EIP4844_FORK_EPOCH=18446744073709551615 +EIP6110_FORK_EPOCH=18446744073709551615 # Spec version (mainnet or minimal) SPEC_PRESET=mainnet diff --git a/testing/ef_tests/src/cases/common.rs b/testing/ef_tests/src/cases/common.rs index f889002bd88..4475f34fbee 100644 --- a/testing/ef_tests/src/cases/common.rs +++ b/testing/ef_tests/src/cases/common.rs @@ -67,6 +67,7 @@ pub fn previous_fork(fork_name: ForkName) -> ForkName { ForkName::Merge => ForkName::Altair, ForkName::Capella => ForkName::Merge, ForkName::Eip4844 => ForkName::Capella, + ForkName::Eip6110 => ForkName::Eip4844, } } diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index 8b04319f648..6685a9682d8 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -104,7 +104,8 @@ impl EpochTransition for JustificationAndFinalization { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => { + | BeaconState::Eip4844(_) + | BeaconState::Eip6110(_) => { let justification_and_finalization_state = altair::process_justification_and_finalization( state, @@ -128,7 +129,8 @@ impl EpochTransition for RewardsAndPenalties { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => altair::process_rewards_and_penalties( + | BeaconState::Eip4844(_) + | BeaconState::Eip6110(_) => altair::process_rewards_and_penalties( state, &altair::ParticipationCache::new(state, spec).unwrap(), spec, @@ -158,7 +160,8 @@ impl EpochTransition for Slashings { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => { + | BeaconState::Eip4844(_) + | BeaconState::Eip6110(_) => { process_slashings( state, altair::ParticipationCache::new(state, spec) @@ -235,7 +238,8 @@ impl EpochTransition for SyncCommitteeUpdates { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => altair::process_sync_committee_updates(state, spec), + | BeaconState::Eip4844(_) + | BeaconState::Eip6110(_) => altair::process_sync_committee_updates(state, spec), } } } @@ -247,7 +251,8 @@ impl EpochTransition for InactivityUpdates { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => altair::process_inactivity_updates( + | BeaconState::Eip4844(_) + | BeaconState::Eip6110(_) => altair::process_inactivity_updates( state, &altair::ParticipationCache::new(state, spec).unwrap(), spec, @@ -263,7 +268,8 @@ impl EpochTransition for ParticipationFlagUpdates { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => altair::process_participation_flag_updates(state), + | BeaconState::Eip4844(_) + | BeaconState::Eip6110(_) => altair::process_participation_flag_updates(state), } } } @@ -318,6 +324,10 @@ impl> Case for EpochProcessing { T::name() != "participation_record_updates" && T::name() != "historical_roots_update" } + ForkName::Eip4844 | ForkName::Eip6110 => { + T::name() != "participation_record_updates" + && T::name() != "historical_roots_update" + } } } diff --git a/testing/ef_tests/src/cases/fork.rs b/testing/ef_tests/src/cases/fork.rs index 26939ce0447..7003e09dcb5 100644 --- a/testing/ef_tests/src/cases/fork.rs +++ b/testing/ef_tests/src/cases/fork.rs @@ -5,6 +5,7 @@ use crate::decode::{ssz_decode_state, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::upgrade::{ upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_eip4844, + upgrade_to_eip6110, }; use types::{BeaconState, ForkName}; @@ -65,6 +66,7 @@ impl Case for ForkTest { ForkName::Merge => upgrade_to_bellatrix(&mut result_state, spec).map(|_| result_state), ForkName::Capella => upgrade_to_capella(&mut result_state, spec).map(|_| result_state), ForkName::Eip4844 => upgrade_to_eip4844(&mut result_state, spec).map(|_| result_state), + ForkName::Eip6110 => upgrade_to_eip6110(&mut result_state, spec).map(|_| result_state), }; compare_beacon_state_results_without_caches(&mut result, &mut expected) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 71954405c0c..f2741167c81 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -98,7 +98,8 @@ impl Operation for Attestation { BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) - | BeaconState::Eip4844(_) => { + | BeaconState::Eip4844(_) + | BeaconState::Eip6110(_) => { altair::process_attestation(state, self, 0, &mut ctxt, VerifySignatures::True, spec) } } diff --git a/testing/ef_tests/src/cases/transition.rs b/testing/ef_tests/src/cases/transition.rs index fb7ccfea644..33b614a4f14 100644 --- a/testing/ef_tests/src/cases/transition.rs +++ b/testing/ef_tests/src/cases/transition.rs @@ -53,6 +53,13 @@ impl LoadCase for TransitionTest { spec.capella_fork_epoch = Some(Epoch::new(0)); spec.eip4844_fork_epoch = Some(metadata.fork_epoch); } + ForkName::Eip6110 => { + spec.altair_fork_epoch = Some(Epoch::new(0)); + spec.bellatrix_fork_epoch = Some(Epoch::new(0)); + spec.capella_fork_epoch = Some(Epoch::new(0)); + spec.eip4844_fork_epoch = Some(Epoch::new(0)); + spec.eip6110_fork_epoch = Some(metadata.fork_epoch); + } } // Load blocks diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index 9a20c3a3e9c..504d9035498 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -222,6 +222,10 @@ impl SszStaticHandler { Self::for_forks(vec![ForkName::Eip4844]) } + pub fn eip6110_only() -> Self { + Self::for_forks(vec![ForkName::Eip6110]) + } + pub fn altair_and_later() -> Self { Self::for_forks(ForkName::list_all()[1..].to_vec()) } diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 19b3535fbfa..29d365a5167 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -48,6 +48,7 @@ type_name_generic!(BeaconBlockBodyAltair, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyMerge, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyCapella, "BeaconBlockBody"); type_name_generic!(BeaconBlockBodyEip4844, "BeaconBlockBody"); +type_name_generic!(BeaconBlockBodyEip6110, "BeaconBlockBody"); type_name!(BeaconBlockHeader); type_name_generic!(BeaconState); type_name_generic!(BlobsSidecar); @@ -61,11 +62,13 @@ type_name_generic!(ExecutionPayload); type_name_generic!(ExecutionPayloadMerge, "ExecutionPayload"); type_name_generic!(ExecutionPayloadCapella, "ExecutionPayload"); type_name_generic!(ExecutionPayloadEip4844, "ExecutionPayload"); +type_name_generic!(ExecutionPayloadEip6110, "ExecutionPayload"); type_name_generic!(FullPayload, "ExecutionPayload"); type_name_generic!(ExecutionPayloadHeader); type_name_generic!(ExecutionPayloadHeaderMerge, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderCapella, "ExecutionPayloadHeader"); type_name_generic!(ExecutionPayloadHeaderEip4844, "ExecutionPayloadHeader"); +type_name_generic!(ExecutionPayloadHeaderEip6110, "ExecutionPayloadHeader"); type_name_generic!(BlindedPayload, "ExecutionPayloadHeader"); type_name!(Fork); type_name!(ForkData); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index fe2c5f796fc..634eff6c291 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -272,6 +272,10 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::eip4844_only() .run(); + SszStaticHandler::, MinimalEthSpec>::eip6110_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::eip6110_only() + .run(); } // Altair and later @@ -336,6 +340,10 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::eip4844_only() .run(); + SszStaticHandler::, MinimalEthSpec>::eip6110_only() + .run(); + SszStaticHandler::, MainnetEthSpec>::eip6110_only() + .run(); } #[test] @@ -352,6 +360,10 @@ mod ssz_static { ::eip4844_only().run(); SszStaticHandler::, MainnetEthSpec> ::eip4844_only().run(); + SszStaticHandler::, MinimalEthSpec> + ::eip6110_only().run(); + SszStaticHandler::, MainnetEthSpec> + ::eip6110_only().run(); } #[test] diff --git a/validator_client/src/signing_method/web3signer.rs b/validator_client/src/signing_method/web3signer.rs index 54352394af2..6edfd83496e 100644 --- a/validator_client/src/signing_method/web3signer.rs +++ b/validator_client/src/signing_method/web3signer.rs @@ -28,6 +28,7 @@ pub enum ForkName { Bellatrix, Capella, Eip4844, + Eip6110, } #[derive(Debug, PartialEq, Serialize)] @@ -101,6 +102,11 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> Web3SignerObject<'a, T, Pa block: None, block_header: Some(block.block_header()), }), + BeaconBlock::Eip6110(_) => Ok(Web3SignerObject::BeaconBlock { + version: ForkName::Eip6110, + block: None, + block_header: Some(block.block_header()), + }), } } From bb983bbf15ded4c4aa657bfb77b4cb8220c9aff7 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Tue, 28 Mar 2023 17:50:08 +0200 Subject: [PATCH 51/96] minor fixes --- .../beacon_chain/src/beacon_block_streamer.rs | 3 ++- beacon_node/execution_layer/src/engine_api.rs | 1 + consensus/types/src/eip6110.rs | 24 +++++++++---------- consensus/types/src/execution_payload.rs | 1 + consensus/types/src/payload.rs | 4 +--- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_block_streamer.rs b/beacon_node/beacon_chain/src/beacon_block_streamer.rs index e56c0c3e399..bb74b368557 100644 --- a/beacon_node/beacon_chain/src/beacon_block_streamer.rs +++ b/beacon_node/beacon_chain/src/beacon_block_streamer.rs @@ -3,7 +3,7 @@ use execution_layer::{ExecutionLayer, ExecutionPayloadBodyV1}; use slog::{crit, debug, Logger}; use std::collections::HashMap; use std::sync::Arc; -use store::{DatabaseBlock, ExecutionPayloadEip4844}; +use store::{DatabaseBlock, ExecutionPayloadEip4844, ExecutionPayloadEip6110}; use task_executor::TaskExecutor; use tokio::sync::{ mpsc::{self, UnboundedSender}, @@ -98,6 +98,7 @@ fn reconstruct_default_header_block( ForkName::Merge => ExecutionPayloadMerge::default().into(), ForkName::Capella => ExecutionPayloadCapella::default().into(), ForkName::Eip4844 => ExecutionPayloadEip4844::default().into(), + ForkName::Eip6110 => ExecutionPayloadEip6110::default().into(), ForkName::Base | ForkName::Altair => { return Err(Error::PayloadReconstruction(format!( "Block with fork variant {} has execution payload", diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 0769a86a382..24544bf640d 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -558,6 +558,7 @@ impl ExecutionPayloadBodyV1 { )) } } + ExecutionPayloadHeader::Eip6110(_) => todo!(), } } } diff --git a/consensus/types/src/eip6110.rs b/consensus/types/src/eip6110.rs index 230db75badb..926e62d251a 100644 --- a/consensus/types/src/eip6110.rs +++ b/consensus/types/src/eip6110.rs @@ -2,22 +2,12 @@ use crate::test_utils::TestRandom; use crate::*; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; +use std::hash::{Hash, Hasher}; use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; #[derive( - arbitrary::Arbitrary, - Debug, - // PartialEq, - // Eq, - Hash, - Clone, - Serialize, - Deserialize, - Encode, - Decode, - TreeHash, - TestRandom, + arbitrary::Arbitrary, Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, )] pub struct DepositReceipt { pub pubkey: PublicKeyBytes, @@ -41,6 +31,16 @@ impl PartialEq for DepositReceipt { } } +// Manually implement the Hash trait for DepositReceipt +impl Hash for DepositReceipt { + fn hash(&self, state: &mut H) { + self.pubkey.hash(state); + self.withdrawal_credentials.hash(state); + self.amount.hash(state); + self.index.hash(state); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 42d3f43839f..3635779b790 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -198,6 +198,7 @@ impl ExecutionPayload { ExecutionPayload::Merge(_) => ForkName::Merge, ExecutionPayload::Capella(_) => ForkName::Capella, ExecutionPayload::Eip4844(_) => ForkName::Eip4844, + ExecutionPayload::Eip6110(_) => ForkName::Eip6110, } } } diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index eeece8f96bd..38e58a6d74c 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -424,9 +424,7 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { // Return an "empty" or "default" value for the EIP-4844 variant Ok(Vec::new().into()) } - FullPayloadRef::Eip6110(ref inner) => { - Ok(inner.execution_payload.deposit_receipts.clone()) - } + FullPayloadRef::Eip6110(inner) => Ok(inner.execution_payload.deposit_receipts.clone()), } } } From 30c8cea79466fdaa2084feaffb2a52e11ecc7589 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Wed, 29 Mar 2023 17:07:42 +0200 Subject: [PATCH 52/96] minor fixes --- .../src/per_block_processing.rs | 4 +- .../src/per_epoch_processing.rs | 6 +- .../src/per_epoch_processing/eip4844.rs | 75 ------------------- .../ef_tests/src/cases/epoch_processing.rs | 2 +- 4 files changed, 6 insertions(+), 81 deletions(-) delete mode 100644 consensus/state_processing/src/per_epoch_processing/eip4844.rs diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 49ecaf23c36..1961c4fea84 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -442,10 +442,10 @@ pub fn process_deposit_receipt( // Create a Deposit object from the DepositReceipt let deposit_data = DepositData { - pubkey: deposit_receipt.pubkey.clone().into(), + pubkey: deposit_receipt.pubkey, withdrawal_credentials: deposit_receipt.withdrawal_credentials, amount: deposit_receipt.amount, - signature: deposit_receipt.signature.clone().into(), + signature: deposit_receipt.signature.clone(), }; let deposit = Deposit { proof: FixedVector::from_elem(Hash256::zero()), // Set an empty proof, since it's not included in DepositReceipt diff --git a/consensus/state_processing/src/per_epoch_processing.rs b/consensus/state_processing/src/per_epoch_processing.rs index 8653b3541cb..765591c647d 100644 --- a/consensus/state_processing/src/per_epoch_processing.rs +++ b/consensus/state_processing/src/per_epoch_processing.rs @@ -14,7 +14,6 @@ pub mod altair; pub mod base; pub mod capella; pub mod effective_balance_updates; -pub mod eip4844; pub mod epoch_processing_summary; pub mod errors; pub mod historical_roots_update; @@ -41,8 +40,9 @@ pub fn process_epoch( match state { BeaconState::Base(_) => base::process_epoch(state, spec), BeaconState::Altair(_) | BeaconState::Merge(_) => altair::process_epoch(state, spec), - BeaconState::Capella(_) | BeaconState::Eip4844(_) => capella::process_epoch(state, spec), - BeaconState::Eip4844(_) | BeaconState::Eip6110(_) => capella::process_epoch(state, spec), + BeaconState::Capella(_) | BeaconState::Eip4844(_) | BeaconState::Eip6110(_) => { + capella::process_epoch(state, spec) + } } } diff --git a/consensus/state_processing/src/per_epoch_processing/eip4844.rs b/consensus/state_processing/src/per_epoch_processing/eip4844.rs deleted file mode 100644 index 5af3758e5b6..00000000000 --- a/consensus/state_processing/src/per_epoch_processing/eip4844.rs +++ /dev/null @@ -1,75 +0,0 @@ -use super::altair::inactivity_updates::process_inactivity_updates; -use super::altair::justification_and_finalization::process_justification_and_finalization; -use super::altair::participation_cache::ParticipationCache; -use super::altair::participation_flag_updates::process_participation_flag_updates; -use super::altair::rewards_and_penalties::process_rewards_and_penalties; -use super::altair::sync_committee_updates::process_sync_committee_updates; -use super::capella::process_historical_summaries_update; -use super::{process_registry_updates, process_slashings, EpochProcessingSummary, Error}; -use crate::per_epoch_processing::{ - effective_balance_updates::process_effective_balance_updates, - resets::{process_eth1_data_reset, process_randao_mixes_reset, process_slashings_reset}, -}; -use types::{BeaconState, ChainSpec, EthSpec, RelativeEpoch}; - -pub fn process_epoch( - state: &mut BeaconState, - spec: &ChainSpec, -) -> Result, Error> { - // Ensure the committee caches are built. - state.build_committee_cache(RelativeEpoch::Previous, spec)?; - state.build_committee_cache(RelativeEpoch::Current, spec)?; - state.build_committee_cache(RelativeEpoch::Next, spec)?; - - // Pre-compute participating indices and total balances. - let participation_cache = ParticipationCache::new(state, spec)?; - let sync_committee = state.current_sync_committee()?.clone(); - - // Justification and finalization. - let justification_and_finalization_state = - process_justification_and_finalization(state, &participation_cache)?; - justification_and_finalization_state.apply_changes_to_state(state); - - process_inactivity_updates(state, &participation_cache, spec)?; - - // Rewards and Penalties. - process_rewards_and_penalties(state, &participation_cache, spec)?; - - // Registry Updates. - process_registry_updates(state, spec)?; - - // Slashings. - process_slashings( - state, - participation_cache.current_epoch_total_active_balance(), - spec, - )?; - - // Reset eth1 data votes. - process_eth1_data_reset(state)?; - - // Update effective balances with hysteresis (lag). - process_effective_balance_updates(state, spec)?; - - // Reset slashings - process_slashings_reset(state)?; - - // Set randao mix - process_randao_mixes_reset(state)?; - - // Set historical summaries accumulator - process_historical_summaries_update(state)?; - - // Rotate current/previous epoch participation - process_participation_flag_updates(state)?; - - process_sync_committee_updates(state, spec)?; - - // Rotate the epoch caches to suit the epoch transition. - state.advance_caches(spec)?; - - Ok(EpochProcessingSummary::Altair { - participation_cache, - sync_committee, - }) -} diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index 6685a9682d8..adbb890af08 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -324,7 +324,7 @@ impl> Case for EpochProcessing { T::name() != "participation_record_updates" && T::name() != "historical_roots_update" } - ForkName::Eip4844 | ForkName::Eip6110 => { + ForkName::Eip6110 => { T::name() != "participation_record_updates" && T::name() != "historical_roots_update" } From 1369a2902dfe63b9af14ad5b9adfbb632a4c3f27 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Wed, 29 Mar 2023 17:43:57 +0200 Subject: [PATCH 53/96] remove tests for `Eip6110` for now --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 711b783aa69..316a990de03 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ PROFILE ?= release # List of all hard forks. This list is used to set env variables for several tests so that # they run for different forks. -FORKS=phase0 altair merge capella eip4844 eip6110 +FORKS=phase0 altair merge capella eip4844 # eip6110 # Extra flags for Cargo CARGO_INSTALL_EXTRA_FLAGS?= From ffd03f85cb7929a4a2510fb9629b85483ce5d71f Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Thu, 30 Mar 2023 16:07:26 +0200 Subject: [PATCH 54/96] clippy --- .../execution_layer/src/engine_api/json_structures.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index c028abdd0e1..d73cd587e5f 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -447,10 +447,10 @@ pub struct JsonDepositReceipt { impl From for JsonDepositReceipt { fn from(deposit_receipt: DepositReceipt) -> Self { Self { - pubkey: deposit_receipt.pubkey.decompress().unwrap().into(), // Convert PublicKeyBytes to PublicKey + pubkey: deposit_receipt.pubkey.decompress().unwrap().into(), withdrawal_credentials: deposit_receipt.withdrawal_credentials, amount: deposit_receipt.amount, - signature: (deposit_receipt.signature).try_into().unwrap(), // Convert SignatureBytes to Signature + signature: deposit_receipt.signature, index: deposit_receipt.index, } } @@ -459,10 +459,10 @@ impl From for JsonDepositReceipt { impl From for DepositReceipt { fn from(json_deposit_receipt: JsonDepositReceipt) -> Self { Self { - pubkey: json_deposit_receipt.pubkey.into(), + pubkey: json_deposit_receipt.pubkey, withdrawal_credentials: json_deposit_receipt.withdrawal_credentials, amount: json_deposit_receipt.amount, - signature: json_deposit_receipt.signature.into(), + signature: json_deposit_receipt.signature, index: json_deposit_receipt.index, } } From d255634002f83fff889c8668759ef1820913f504 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Thu, 30 Mar 2023 16:16:39 +0200 Subject: [PATCH 55/96] fix Makefile --- Makefile | 2 +- testing/ef_tests/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 316a990de03..bf2ad679453 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ PROFILE ?= release # List of all hard forks. This list is used to set env variables for several tests so that # they run for different forks. -FORKS=phase0 altair merge capella eip4844 # eip6110 +FORKS=phase0 altair merge capella eip4844 # Extra flags for Cargo CARGO_INSTALL_EXTRA_FLAGS?= diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index eb2aa10307e..1feba41c86f 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,4 +1,4 @@ -TESTS_TAG := v1.3.0-rc.1 # FIXME: move to latest +TESTS_TAG := v1.3.0-rc.1 TESTS = general minimal mainnet TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) From 38f8b54a12c476bd65c71d309e791d4ad6752b6e Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Mon, 3 Apr 2023 11:27:21 +0200 Subject: [PATCH 56/96] `V4` -> `V6110` --- .../execution_layer/src/engine_api/http.rs | 14 ++++----- .../src/engine_api/json_structures.rs | 31 ++++++++++--------- .../src/test_utils/handle_rpc.rs | 16 +++++----- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 505eb11cd61..5b6c7b2cc04 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -33,13 +33,13 @@ pub const ETH_SYNCING_TIMEOUT: Duration = Duration::from_secs(1); pub const ENGINE_NEW_PAYLOAD_V1: &str = "engine_newPayloadV1"; pub const ENGINE_NEW_PAYLOAD_V2: &str = "engine_newPayloadV2"; pub const ENGINE_NEW_PAYLOAD_V3: &str = "engine_newPayloadV3"; -pub const ENGINE_NEW_PAYLOAD_V4: &str = "engine_newPayloadV4"; +pub const ENGINE_NEW_PAYLOAD_V6110: &str = "engine_newPayloadV6110"; pub const ENGINE_NEW_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(8); pub const ENGINE_GET_PAYLOAD_V1: &str = "engine_getPayloadV1"; pub const ENGINE_GET_PAYLOAD_V2: &str = "engine_getPayloadV2"; pub const ENGINE_GET_PAYLOAD_V3: &str = "engine_getPayloadV3"; -pub const ENGINE_GET_PAYLOAD_V4: &str = "engine_getPayloadV4"; +pub const ENGINE_GET_PAYLOAD_V6110: &str = "engine_getPayloadV6110"; pub const ENGINE_GET_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(2); pub const ENGINE_GET_BLOBS_BUNDLE_V1: &str = "engine_getBlobsBundleV1"; @@ -897,14 +897,14 @@ impl HttpJsonRpc { Ok(JsonGetPayloadResponse::V3(response).into()) } ForkName::Eip6110 => { - let response: JsonGetPayloadResponseV4 = self + let response: JsonGetPayloadResponseV6110 = self .rpc_request( ENGINE_GET_PAYLOAD_V2, params, ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, ) .await?; - Ok(JsonGetPayloadResponse::V4(response).into()) + Ok(JsonGetPayloadResponse::V6110(response).into()) } ForkName::Base | ForkName::Altair => Err(Error::UnsupportedForkVariant(format!( "called get_payload_v2 with {}", @@ -952,14 +952,14 @@ impl HttpJsonRpc { Ok(JsonGetPayloadResponse::V3(response).into()) } ForkName::Eip6110 => { - let response: JsonGetPayloadResponseV4 = self + let response: JsonGetPayloadResponseV6110 = self .rpc_request( - ENGINE_GET_PAYLOAD_V4, + ENGINE_GET_PAYLOAD_V6110, params, ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, ) .await?; - Ok(JsonGetPayloadResponse::V4(response).into()) + Ok(JsonGetPayloadResponse::V6110(response).into()) } ForkName::Base | ForkName::Altair => Err(Error::UnsupportedForkVariant(format!( "called get_payload_v3 with {}", diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index d73cd587e5f..921aef4a2bd 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -65,7 +65,7 @@ pub struct JsonPayloadIdResponse { } #[superstruct( - variants(V1, V2, V3, V4), + variants(V1, V2, V3, V6110), variant_attributes( derive(Debug, PartialEq, Default, Serialize, Deserialize,), serde(bound = "T: EthSpec", rename_all = "camelCase"), @@ -95,16 +95,17 @@ pub struct JsonExecutionPayload { pub extra_data: VariableList, #[serde(with = "eth2_serde_utils::u256_hex_be")] pub base_fee_per_gas: Uint256, - #[superstruct(only(V3, V4))] + #[superstruct(only(V3, V6110))] #[serde(with = "eth2_serde_utils::u256_hex_be")] pub excess_data_gas: Uint256, pub block_hash: ExecutionBlockHash, #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: VariableList, T::MaxTransactionsPerPayload>, - #[superstruct(only(V2, V3, V4))] + #[superstruct(only(V2, V3, V6110))] pub withdrawals: VariableList, - #[superstruct(only(V4))] - pub deposit_receipts: VariableList, + #[superstruct(only(V6110))] + pub deposit_receipts: VariableList, + } impl From> for JsonExecutionPayloadV1 { @@ -180,9 +181,9 @@ impl From> for JsonExecutionPayloadV3 } } } -impl From> for JsonExecutionPayloadV4 { +impl From> for JsonExecutionPayloadV6110 { fn from(payload: ExecutionPayloadEip6110) -> Self { - JsonExecutionPayloadV4 { + JsonExecutionPayloadV6110 { parent_hash: payload.parent_hash, fee_recipient: payload.fee_recipient, state_root: payload.state_root, @@ -220,7 +221,7 @@ impl From> for JsonExecutionPayload { ExecutionPayload::Merge(payload) => JsonExecutionPayload::V1(payload.into()), ExecutionPayload::Capella(payload) => JsonExecutionPayload::V2(payload.into()), ExecutionPayload::Eip4844(payload) => JsonExecutionPayload::V3(payload.into()), - ExecutionPayload::Eip6110(payload) => JsonExecutionPayload::V4(payload.into()), + ExecutionPayload::Eip6110(payload) => JsonExecutionPayload::V6110(payload.into()), } } } @@ -298,8 +299,8 @@ impl From> for ExecutionPayloadEip4844 } } } -impl From> for ExecutionPayloadEip6110 { - fn from(payload: JsonExecutionPayloadV4) -> Self { +impl From> for ExecutionPayloadEip6110 { + fn from(payload: JsonExecutionPayloadV6110) -> Self { ExecutionPayloadEip6110 { parent_hash: payload.parent_hash, fee_recipient: payload.fee_recipient, @@ -338,13 +339,13 @@ impl From> for ExecutionPayload { JsonExecutionPayload::V1(payload) => ExecutionPayload::Merge(payload.into()), JsonExecutionPayload::V2(payload) => ExecutionPayload::Capella(payload.into()), JsonExecutionPayload::V3(payload) => ExecutionPayload::Eip4844(payload.into()), - JsonExecutionPayload::V4(payload) => ExecutionPayload::Eip6110(payload.into()), + JsonExecutionPayload::V6110(payload) => ExecutionPayload::Eip6110(payload.into()), } } } #[superstruct( - variants(V1, V2, V3, V4), + variants(V1, V2, V3, V6110), variant_attributes( derive(Debug, PartialEq, Serialize, Deserialize), serde(bound = "T: EthSpec", rename_all = "camelCase") @@ -361,8 +362,8 @@ pub struct JsonGetPayloadResponse { pub execution_payload: JsonExecutionPayloadV2, #[superstruct(only(V3), partial_getter(rename = "execution_payload_v3"))] pub execution_payload: JsonExecutionPayloadV3, - #[superstruct(only(V4), partial_getter(rename = "execution_payload_v4"))] - pub execution_payload: JsonExecutionPayloadV4, + #[superstruct(only(V6110), partial_getter(rename = "execution_payload_v6110"))] + pub execution_payload: JsonExecutionPayloadV6110, #[serde(with = "eth2_serde_utils::u256_hex_be")] pub block_value: Uint256, } @@ -388,7 +389,7 @@ impl From> for GetPayloadResponse { block_value: response.block_value, }) } - JsonGetPayloadResponse::V4(response) => { + JsonGetPayloadResponse::V6110(response) => { GetPayloadResponse::Eip6110(GetPayloadResponseEip6110 { execution_payload: response.execution_payload.into(), block_value: response.block_value, diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 3c483bdb8fb..9dcd03a60f4 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -112,8 +112,8 @@ pub async fn handle_rpc( }) }) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?, - ENGINE_NEW_PAYLOAD_V4 => get_param::>(params, 0) - .map(|jep| JsonExecutionPayload::V4(jep)) + ENGINE_NEW_PAYLOAD_V6110 => get_param::>(params, 0) + .map(|jep| JsonExecutionPayload::V6110(jep)) .or_else(|_| { get_param::>(params, 0) .map(|jep| JsonExecutionPayload::V3(jep)) @@ -264,7 +264,7 @@ pub async fn handle_rpc( ENGINE_GET_PAYLOAD_V1 | ENGINE_GET_PAYLOAD_V2 | ENGINE_GET_PAYLOAD_V3 - | ENGINE_GET_PAYLOAD_V4 => { + | ENGINE_GET_PAYLOAD_V6110 => { let request: JsonPayloadIdRequest = get_param(params, 0).map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?; let id = request.into(); @@ -350,15 +350,15 @@ pub async fn handle_rpc( }) .unwrap() } - JsonExecutionPayload::V4(execution_payload) => { - serde_json::to_value(JsonGetPayloadResponseV4 { + JsonExecutionPayload::V6110(execution_payload) => { + serde_json::to_value(JsonGetPayloadResponseV6110 { execution_payload, block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), }) .unwrap() } }), - ENGINE_GET_PAYLOAD_V4 => Ok(match JsonExecutionPayload::from(response) { + ENGINE_GET_PAYLOAD_V6110 => Ok(match JsonExecutionPayload::from(response) { JsonExecutionPayload::V1(execution_payload) => { serde_json::to_value(JsonGetPayloadResponseV1 { execution_payload, @@ -380,8 +380,8 @@ pub async fn handle_rpc( }) .unwrap() } - JsonExecutionPayload::V4(execution_payload) => { - serde_json::to_value(JsonGetPayloadResponseV4 { + JsonExecutionPayload::V6110(execution_payload) => { + serde_json::to_value(JsonGetPayloadResponseV6110 { execution_payload, block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), }) From f733a23a2f95961af0e53952c0cd0276511b41a5 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Mon, 10 Apr 2023 10:27:12 +0200 Subject: [PATCH 57/96] Fix `JsonPayloadAttributes` --- beacon_node/execution_layer/src/engine_api.rs | 6 +----- .../execution_layer/src/engine_api/json_structures.rs | 4 ---- .../src/test_utils/execution_block_generator.rs | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 24544bf640d..8750a9bf3a5 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -22,7 +22,7 @@ pub use types::{ Withdrawal, Withdrawals, }; use types::{ - DepositReceipt, ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadEip6110, + ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadEip6110, ExecutionPayloadMerge, }; @@ -327,8 +327,6 @@ pub struct PayloadAttributes { pub suggested_fee_recipient: Address, #[superstruct(only(V2))] pub withdrawals: Vec, - #[superstruct(only(V2))] - pub deposit_receipts: Vec, } impl PayloadAttributes { @@ -344,7 +342,6 @@ impl PayloadAttributes { prev_randao, suggested_fee_recipient, withdrawals, - deposit_receipts: Vec::new(), }), None => PayloadAttributes::V1(PayloadAttributesV1 { timestamp, @@ -372,7 +369,6 @@ impl From for SsePayloadAttributes { prev_randao, suggested_fee_recipient, withdrawals, - deposit_receipts: _, }) => Self::V2(SsePayloadAttributesV2 { timestamp, prev_randao, diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 921aef4a2bd..82423746965 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -487,8 +487,6 @@ pub struct JsonPayloadAttributes { pub suggested_fee_recipient: Address, #[superstruct(only(V2))] pub withdrawals: Vec, - #[superstruct(only(V2))] - pub deposit_receipts: Vec, } impl From for JsonPayloadAttributes { @@ -504,7 +502,6 @@ impl From for JsonPayloadAttributes { prev_randao: pa.prev_randao, suggested_fee_recipient: pa.suggested_fee_recipient, withdrawals: pa.withdrawals.into_iter().map(Into::into).collect(), - deposit_receipts: pa.deposit_receipts.into_iter().map(Into::into).collect(), }), } } @@ -523,7 +520,6 @@ impl From for PayloadAttributes { prev_randao: jpa.prev_randao, suggested_fee_recipient: jpa.suggested_fee_recipient, withdrawals: jpa.withdrawals.into_iter().map(Into::into).collect(), - deposit_receipts: jpa.deposit_receipts.into_iter().map(Into::into).collect(), }), } } diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 768c2d35dc5..99df3cbf3dc 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -582,7 +582,7 @@ impl ExecutionBlockGenerator { block_hash: ExecutionBlockHash::zero(), transactions: vec![].into(), withdrawals: pa.withdrawals.clone().into(), - deposit_receipts: pa.deposit_receipts.clone().into(), + deposit_receipts: vec![].into(), }) } _ => unreachable!(), From 6bc054beb86e97b60f0315af7a3b4a1646f586e9 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Tue, 11 Apr 2023 12:27:46 +0200 Subject: [PATCH 58/96] add consensus-spec-tests of `deposit_receipt` --- testing/ef_tests/src/cases/operations.rs | 31 ++++++++++++++++++++++-- testing/ef_tests/src/type_name.rs | 1 + testing/ef_tests/tests/tests.rs | 6 +++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index f2741167c81..fc0366a48a0 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -7,7 +7,7 @@ use serde_derive::Deserialize; use state_processing::{ per_block_processing::{ errors::BlockProcessingError, - process_block_header, process_execution_payload, + process_block_header, process_deposit_receipt, process_execution_payload, process_operations::{ altair, base, process_attester_slashings, process_bls_to_execution_changes, process_deposits, process_exits, process_proposer_slashings, @@ -19,7 +19,7 @@ use state_processing::{ use std::fmt::Debug; use std::path::Path; use types::{ - Attestation, AttesterSlashing, BeaconBlock, BeaconState, BlindedPayload, ChainSpec, Deposit, + Attestation, AttesterSlashing, BeaconBlock, BeaconState, BlindedPayload, ChainSpec, Deposit, DepositReceipt, EthSpec, ExecutionPayload, ForkName, FullPayload, ProposerSlashing, SignedBlsToExecutionChange, SignedVoluntaryExit, SyncAggregate, }; @@ -367,6 +367,33 @@ impl Operation for WithdrawalsPayload { } } +impl Operation for DepositReceipt { + fn handler_name() -> String { + "deposit_receipt".into() + } + + fn filename() -> String { + "deposit_receipt.ssz_snappy".into() + } + + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name != ForkName::Base && fork_name != ForkName::Altair && fork_name != ForkName::Merge && fork_name != ForkName::Capella && fork_name != ForkName::Eip4844 + } + + fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { + ssz_decode_file(path) + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + _: &Operations, + ) -> Result<(), BlockProcessingError> { + process_deposit_receipt(state, self, spec) + } + } + impl Operation for SignedBlsToExecutionChange { fn handler_name() -> String { "bls_to_execution_change".into() diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 29d365a5167..07af7ccc9a3 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -55,6 +55,7 @@ type_name_generic!(BlobsSidecar); type_name!(Checkpoint); type_name_generic!(ContributionAndProof); type_name!(Deposit); +type_name!(DepositReceipt); type_name!(DepositData); type_name!(DepositMessage); type_name!(Eth1Data); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 634eff6c291..acaf38c667e 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -34,6 +34,12 @@ fn operations_deposit() { OperationsHandler::::default().run(); } +#[test] +fn operations_deposit_receipt() { + OperationsHandler::::default().run(); + OperationsHandler::::default().run(); +} + #[test] fn operations_exit() { OperationsHandler::::default().run(); From 850988cec82abd73e104e6c0d37f5f97dbcab8fa Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Tue, 11 Apr 2023 13:43:13 +0200 Subject: [PATCH 59/96] fmt and clippy :< --- .../execution_layer/src/test_utils/mod.rs | 1 + testing/ef_tests/src/cases/operations.rs | 26 +++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index e1a148caaa5..3428f9d91cb 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -173,6 +173,7 @@ impl MockServer { *self.ctx.engine_capabilities.write() = engine_capabilities; } + #[allow(clippy::too_many_arguments)] // FIXME: refactor pub fn new( handle: &runtime::Handle, jwt_key: JwtKey, diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index fc0366a48a0..ebb5142d685 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -19,9 +19,9 @@ use state_processing::{ use std::fmt::Debug; use std::path::Path; use types::{ - Attestation, AttesterSlashing, BeaconBlock, BeaconState, BlindedPayload, ChainSpec, Deposit, DepositReceipt, - EthSpec, ExecutionPayload, ForkName, FullPayload, ProposerSlashing, SignedBlsToExecutionChange, - SignedVoluntaryExit, SyncAggregate, + Attestation, AttesterSlashing, BeaconBlock, BeaconState, BlindedPayload, ChainSpec, Deposit, + DepositReceipt, EthSpec, ExecutionPayload, ForkName, FullPayload, ProposerSlashing, + SignedBlsToExecutionChange, SignedVoluntaryExit, SyncAggregate, }; #[derive(Debug, Clone, Default, Deserialize)] @@ -377,7 +377,11 @@ impl Operation for DepositReceipt { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name != ForkName::Base && fork_name != ForkName::Altair && fork_name != ForkName::Merge && fork_name != ForkName::Capella && fork_name != ForkName::Eip4844 + fork_name != ForkName::Base + && fork_name != ForkName::Altair + && fork_name != ForkName::Merge + && fork_name != ForkName::Capella + && fork_name != ForkName::Eip4844 } fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -385,14 +389,14 @@ impl Operation for DepositReceipt { } fn apply_to( - &self, - state: &mut BeaconState, - spec: &ChainSpec, - _: &Operations, - ) -> Result<(), BlockProcessingError> { - process_deposit_receipt(state, self, spec) - } + &self, + state: &mut BeaconState, + spec: &ChainSpec, + _: &Operations, + ) -> Result<(), BlockProcessingError> { + process_deposit_receipt(state, self, spec) } +} impl Operation for SignedBlsToExecutionChange { fn handler_name() -> String { From d2731f061c6a65b287a6170a00f947c207cf542a Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Wed, 12 Apr 2023 11:53:27 +0200 Subject: [PATCH 60/96] minor fix --- consensus/state_processing/src/per_block_processing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 1961c4fea84..8b9be9f576f 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -453,7 +453,7 @@ pub fn process_deposit_receipt( }; // Call process_deposit with the created Deposit object - process_deposit(state, &deposit, spec, true)?; + process_deposit(state, &deposit, spec, false)?; Ok(()) } From f5c97fd291c955ef6ed47a80e1b44592659f4066 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Wed, 12 Apr 2023 12:31:17 +0200 Subject: [PATCH 61/96] cargo fmt --- .../execution_layer/src/engine_api/json_structures.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 82423746965..4e057a21779 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -7,8 +7,8 @@ use superstruct::superstruct; use types::blobs_sidecar::KzgCommitments; use types::{ Blobs, DepositReceipt, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, - ExecutionPayloadEip4844, ExecutionPayloadEip6110, ExecutionPayloadMerge, FixedVector, Transactions, Transaction, Unsigned, - VariableList, Withdrawal, + ExecutionPayloadEip4844, ExecutionPayloadEip6110, ExecutionPayloadMerge, FixedVector, + Transaction, Transactions, Unsigned, VariableList, Withdrawal, }; #[derive(Debug, PartialEq, Serialize, Deserialize)] @@ -100,12 +100,12 @@ pub struct JsonExecutionPayload { pub excess_data_gas: Uint256, pub block_hash: ExecutionBlockHash, #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] - pub transactions: VariableList, T::MaxTransactionsPerPayload>, + pub transactions: + VariableList, T::MaxTransactionsPerPayload>, #[superstruct(only(V2, V3, V6110))] pub withdrawals: VariableList, #[superstruct(only(V6110))] pub deposit_receipts: VariableList, - } impl From> for JsonExecutionPayloadV1 { From c5e5e85edcc5e78307e74c943c307e6071d5f8dc Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Wed, 12 Apr 2023 17:34:51 +0200 Subject: [PATCH 62/96] fix `operations_deposit_receipt` test --- consensus/state_processing/src/per_block_processing.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 8b9be9f576f..b05b6a7885d 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -452,9 +452,15 @@ pub fn process_deposit_receipt( data: deposit_data, }; + // Store the current eth1_deposit_index + let current_eth1_deposit_index = state.eth1_deposit_index(); + // Call process_deposit with the created Deposit object process_deposit(state, &deposit, spec, false)?; + // Set eth1_deposit_index back to the original value + *state.eth1_deposit_index_mut() = current_eth1_deposit_index; + Ok(()) } From 5db79a92013c13c5c854842a889b178981fc2ec8 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Fri, 14 Apr 2023 16:29:33 +0200 Subject: [PATCH 63/96] cleanup --- beacon_node/execution_layer/src/engine_api.rs | 2 + .../execution_layer/src/engine_api/http.rs | 53 ++++++++++++++----- beacon_node/execution_layer/src/lib.rs | 1 - .../execution_layer/src/test_utils/mod.rs | 2 + consensus/fork_choice/src/fork_choice.rs | 4 +- .../src/per_block_processing.rs | 2 +- .../src/per_block_processing/eip6110.rs | 2 - .../per_block_processing/eip6110/eip6110.rs | 1 - .../process_operations.rs | 23 ++++++-- consensus/types/src/beacon_block.rs | 6 +-- consensus/types/src/beacon_block_body.rs | 1 - consensus/types/src/beacon_state.rs | 1 - 12 files changed, 69 insertions(+), 29 deletions(-) delete mode 100644 consensus/state_processing/src/per_block_processing/eip6110.rs delete mode 100644 consensus/state_processing/src/per_block_processing/eip6110/eip6110.rs diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 8750a9bf3a5..09be765fa4e 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -564,6 +564,7 @@ pub struct EngineCapabilities { pub new_payload_v1: bool, pub new_payload_v2: bool, pub new_payload_v3: bool, + pub new_payload_v6110: bool, pub forkchoice_updated_v1: bool, pub forkchoice_updated_v2: bool, pub get_payload_bodies_by_hash_v1: bool, @@ -571,6 +572,7 @@ pub struct EngineCapabilities { pub get_payload_v1: bool, pub get_payload_v2: bool, pub get_payload_v3: bool, + pub get_payload_v6110: bool, pub exchange_transition_configuration_v1: bool, } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 5b6c7b2cc04..9eadb44d77e 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -88,6 +88,7 @@ pub static PRE_CAPELLA_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilit new_payload_v1: true, new_payload_v2: false, new_payload_v3: false, + new_payload_v6110: false, forkchoice_updated_v1: true, forkchoice_updated_v2: false, get_payload_bodies_by_hash_v1: false, @@ -95,6 +96,7 @@ pub static PRE_CAPELLA_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilit get_payload_v1: true, get_payload_v2: false, get_payload_v3: false, + get_payload_v6110: false, exchange_transition_configuration_v1: true, }; @@ -835,6 +837,23 @@ impl HttpJsonRpc { Ok(response.into()) } + pub async fn new_payload_v6110( + &self, + execution_payload: ExecutionPayload, + ) -> Result { + let params = json!([JsonExecutionPayload::from(execution_payload)]); + + let response: JsonPayloadStatusV1 = self + .rpc_request( + ENGINE_NEW_PAYLOAD_V6110, + params, + ENGINE_NEW_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + + Ok(response.into()) + } + pub async fn get_payload_v1( &self, payload_id: PayloadId, @@ -919,7 +938,7 @@ impl HttpJsonRpc { payload_id: PayloadId, ) -> Result, Error> { let params = json!([JsonPayloadIdRequest::from(payload_id)]); - + match fork_name { ForkName::Merge => { let response: JsonGetPayloadResponseV1 = self @@ -951,22 +970,30 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V3(response).into()) } - ForkName::Eip6110 => { - let response: JsonGetPayloadResponseV6110 = self - .rpc_request( - ENGINE_GET_PAYLOAD_V6110, - params, - ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, - ) - .await?; - Ok(JsonGetPayloadResponse::V6110(response).into()) - } + ForkName::Eip6110 => self.get_payload_v6110(payload_id).await, ForkName::Base | ForkName::Altair => Err(Error::UnsupportedForkVariant(format!( "called get_payload_v3 with {}", fork_name ))), } - } + } + + pub async fn get_payload_v6110( + &self, + payload_id: PayloadId, + ) -> Result, Error> { + let params = json!([JsonPayloadIdRequest::from(payload_id)]); + + let response: JsonGetPayloadResponseV6110 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V6110, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + + Ok(JsonGetPayloadResponse::V6110(response).into()) + } pub async fn get_blobs_bundle_v1( &self, @@ -1112,6 +1139,7 @@ impl HttpJsonRpc { new_payload_v1: capabilities.contains(ENGINE_NEW_PAYLOAD_V1), new_payload_v2: capabilities.contains(ENGINE_NEW_PAYLOAD_V2), new_payload_v3: capabilities.contains(ENGINE_NEW_PAYLOAD_V3), + new_payload_v6110: capabilities.contains(ENGINE_NEW_PAYLOAD_V6110), forkchoice_updated_v1: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V1), forkchoice_updated_v2: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V2), get_payload_bodies_by_hash_v1: capabilities @@ -1121,6 +1149,7 @@ impl HttpJsonRpc { get_payload_v1: capabilities.contains(ENGINE_GET_PAYLOAD_V1), get_payload_v2: capabilities.contains(ENGINE_GET_PAYLOAD_V2), get_payload_v3: capabilities.contains(ENGINE_GET_PAYLOAD_V3), + get_payload_v6110: capabilities.contains(ENGINE_GET_PAYLOAD_V6110), exchange_transition_configuration_v1: capabilities .contains(ENGINE_EXCHANGE_TRANSITION_CONFIGURATION_V1), }), diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 4a7ea6ccdb7..8a7d15703b2 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1792,7 +1792,6 @@ impl ExecutionLayer { .collect(), ) .map_err(ApiError::DeserializeWithdrawals)?; - ExecutionPayload::Eip4844(ExecutionPayloadEip4844 { parent_hash: eip4844_block.parent_hash, fee_recipient: eip4844_block.fee_recipient, diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 3428f9d91cb..e4b7579f289 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -38,6 +38,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { new_payload_v1: true, new_payload_v2: true, new_payload_v3: true, + new_payload_v6110: true, forkchoice_updated_v1: true, forkchoice_updated_v2: true, get_payload_bodies_by_hash_v1: true, @@ -45,6 +46,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { get_payload_v1: true, get_payload_v2: true, get_payload_v3: true, + get_payload_v6110: true, exchange_transition_configuration_v1: true, }; diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index cf8407934c5..1839925f58b 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -753,8 +753,8 @@ where let justification_and_finalization_state = match block { // TODO(eip4844): Ensure that the final specification // does not substantially modify per epoch processing. - BeaconBlockRef::Eip4844(_) - | BeaconBlockRef::Eip6110(_) + BeaconBlockRef::Eip6110(_) + | BeaconBlockRef::Eip4844(_) | BeaconBlockRef::Capella(_) | BeaconBlockRef::Merge(_) | BeaconBlockRef::Altair(_) => { diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index b05b6a7885d..016410fa477 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -29,7 +29,6 @@ pub use verify_exit::verify_exit; pub mod altair; pub mod block_signature_verifier; pub mod eip4844; -pub mod eip6110; pub mod errors; mod is_valid_indexed_attestation; pub mod process_operations; @@ -455,6 +454,7 @@ pub fn process_deposit_receipt( // Store the current eth1_deposit_index let current_eth1_deposit_index = state.eth1_deposit_index(); + // FIXME: This is a workaround to apply_deposit() // Call process_deposit with the created Deposit object process_deposit(state, &deposit, spec, false)?; diff --git a/consensus/state_processing/src/per_block_processing/eip6110.rs b/consensus/state_processing/src/per_block_processing/eip6110.rs deleted file mode 100644 index 3358e302a60..00000000000 --- a/consensus/state_processing/src/per_block_processing/eip6110.rs +++ /dev/null @@ -1,2 +0,0 @@ -#[allow(clippy::module_inception)] -pub mod eip6110; diff --git a/consensus/state_processing/src/per_block_processing/eip6110/eip6110.rs b/consensus/state_processing/src/per_block_processing/eip6110/eip6110.rs deleted file mode 100644 index 8b137891791..00000000000 --- a/consensus/state_processing/src/per_block_processing/eip6110/eip6110.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index bb9fc9bf5ac..78c0b6f9d72 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -21,6 +21,23 @@ pub fn process_operations>( .deposit_count .saturating_sub(state.eth1_deposit_index()); let max_deposits = ::MaxDeposits::to_u64(); + let eth1_deposit_index_limit = state + .deposit_receipts_start_index() + .unwrap_or_else(|_| state.eth1_data().deposit_count); + + if state.eth1_deposit_index() < eth1_deposit_index_limit { + assert_eq!( + block_body.deposits().len(), + std::cmp::min(max_deposits as usize, unprocessed_deposits_count as usize), + "Number of deposits in block does not match the minimum of the maximum number of deposits and the number of unprocessed deposits" + ); + } else { + assert_eq!( + block_body.deposits().len(), + 0, + "Number of deposits in block must be zero when all prior deposits are processed" + ); + } process_proposer_slashings( state, @@ -37,11 +54,6 @@ pub fn process_operations>( spec, )?; process_attestations(state, block_body, verify_signatures, ctxt, spec)?; - assert_eq!( - block_body.deposits().len(), - std::cmp::min(max_deposits as usize, unprocessed_deposits_count as usize), - "Number of deposits in block does not match the minimum of the maximum number of deposits and the number of unprocessed deposits" - ); process_deposits(state, block_body.deposits(), spec)?; process_exits(state, block_body.voluntary_exits(), verify_signatures, spec)?; @@ -57,6 +69,7 @@ pub fn process_operations>( Ok(()) } + pub mod base { use super::*; diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 7d2a1f19888..04a315f2309 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -128,13 +128,13 @@ impl> BeaconBlock { /// Usually it's better to prefer `from_ssz_bytes` which will decode the correct variant based /// on the fork slot. pub fn any_from_ssz_bytes(bytes: &[u8]) -> Result { - BeaconBlockEip4844::from_ssz_bytes(bytes) - .map(BeaconBlock::Eip4844) + BeaconBlockEip6110::from_ssz_bytes(bytes) + .map(BeaconBlock::Eip6110) .or_else(|_| BeaconBlockCapella::from_ssz_bytes(bytes).map(BeaconBlock::Capella)) .or_else(|_| BeaconBlockMerge::from_ssz_bytes(bytes).map(BeaconBlock::Merge)) .or_else(|_| BeaconBlockAltair::from_ssz_bytes(bytes).map(BeaconBlock::Altair)) .or_else(|_| BeaconBlockBase::from_ssz_bytes(bytes).map(BeaconBlock::Base)) - .or_else(|_| BeaconBlockEip6110::from_ssz_bytes(bytes).map(BeaconBlock::Eip6110)) + .or_else(|_| BeaconBlockEip4844::from_ssz_bytes(bytes).map(BeaconBlock::Eip4844)) } /// Convenience accessor for the `body` as a `BeaconBlockBodyRef`. diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index 0506763a925..54cd730fedc 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -1,4 +1,3 @@ -use crate::payload::{BlindedPayloadEip6110, FullPayloadEip6110}; use crate::*; use crate::{blobs_sidecar::KzgCommitments, test_utils::TestRandom}; use derivative::Derivative; diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 7f7311b81e2..f82b9884fd8 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -1,6 +1,5 @@ use self::committee_cache::get_active_validator_indices; use self::exit_cache::ExitCache; -use crate::execution_payload_header::ExecutionPayloadHeaderEip6110; use crate::test_utils::TestRandom; use crate::*; use compare_fields::CompareFields; From 8c3f0d3edb591220ab803018786a85d454b4126e Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Fri, 14 Apr 2023 16:31:20 +0200 Subject: [PATCH 64/96] cargo fmt --- beacon_node/execution_layer/src/engine_api/http.rs | 10 +++++----- .../src/per_block_processing/process_operations.rs | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 9eadb44d77e..64558bfcd87 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -938,7 +938,7 @@ impl HttpJsonRpc { payload_id: PayloadId, ) -> Result, Error> { let params = json!([JsonPayloadIdRequest::from(payload_id)]); - + match fork_name { ForkName::Merge => { let response: JsonGetPayloadResponseV1 = self @@ -976,14 +976,14 @@ impl HttpJsonRpc { fork_name ))), } - } + } pub async fn get_payload_v6110( &self, payload_id: PayloadId, ) -> Result, Error> { let params = json!([JsonPayloadIdRequest::from(payload_id)]); - + let response: JsonGetPayloadResponseV6110 = self .rpc_request( ENGINE_GET_PAYLOAD_V6110, @@ -991,9 +991,9 @@ impl HttpJsonRpc { ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, ) .await?; - + Ok(JsonGetPayloadResponse::V6110(response).into()) - } + } pub async fn get_blobs_bundle_v1( &self, diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 78c0b6f9d72..5999df06d74 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -69,7 +69,6 @@ pub fn process_operations>( Ok(()) } - pub mod base { use super::*; From 5070adda43a249fe4d8d973b436d0a582fff7247 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Fri, 14 Apr 2023 17:45:16 +0200 Subject: [PATCH 65/96] add `apply_deposit()` --- .../src/per_block_processing.rs | 13 ++--- .../process_operations.rs | 48 +++++++++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 016410fa477..e866d3b140d 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -7,7 +7,7 @@ use std::borrow::Cow; use tree_hash::TreeHash; use types::*; -use self::process_operations::process_deposit; +use self::process_operations::apply_deposit; pub use self::verify_attester_slashing::{ get_slashable_indices, get_slashable_indices_modular, verify_attester_slashing, }; @@ -451,15 +451,8 @@ pub fn process_deposit_receipt( data: deposit_data, }; - // Store the current eth1_deposit_index - let current_eth1_deposit_index = state.eth1_deposit_index(); - - // FIXME: This is a workaround to apply_deposit() - // Call process_deposit with the created Deposit object - process_deposit(state, &deposit, spec, false)?; - - // Set eth1_deposit_index back to the original value - *state.eth1_deposit_index_mut() = current_eth1_deposit_index; + // Call apply with the created Deposit object + apply_deposit(state, &deposit, spec)?; Ok(()) } diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 5999df06d74..3d32877b8c1 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -450,3 +450,51 @@ pub fn process_deposit( Ok(()) } + +pub fn apply_deposit( + state: &mut BeaconState, + deposit: &Deposit, + spec: &ChainSpec, +) -> Result<(), BlockProcessingError> { + let amount = deposit.data.amount; + + if let Ok(Some(index)) = get_existing_validator_index(state, &deposit.data.pubkey) { + // Update the existing validator balance. + increase_balance(state, index as usize, amount)?; + } else { + // The signature should be checked for new validators. Return early for a bad signature. + if verify_deposit_signature(&deposit.data, spec).is_err() { + return Ok(()); + } + + // Create a new validator. + let validator = Validator { + pubkey: deposit.data.pubkey, + withdrawal_credentials: deposit.data.withdrawal_credentials, + activation_eligibility_epoch: spec.far_future_epoch, + activation_epoch: spec.far_future_epoch, + exit_epoch: spec.far_future_epoch, + withdrawable_epoch: spec.far_future_epoch, + effective_balance: std::cmp::min( + amount.safe_sub(amount.safe_rem(spec.effective_balance_increment)?)?, + spec.max_effective_balance, + ), + slashed: false, + }; + state.validators_mut().push(validator)?; + state.balances_mut().push(deposit.data.amount)?; + + // Altair or later initializations. + if let Ok(previous_epoch_participation) = state.previous_epoch_participation_mut() { + previous_epoch_participation.push(ParticipationFlags::default())?; + } + if let Ok(current_epoch_participation) = state.current_epoch_participation_mut() { + current_epoch_participation.push(ParticipationFlags::default())?; + } + if let Ok(inactivity_scores) = state.inactivity_scores_mut() { + inactivity_scores.push(0)?; + } + } + + Ok(()) +} From 732326eb5e822cc30d5875af038d8d983c0b5170 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Fri, 14 Apr 2023 18:25:40 +0200 Subject: [PATCH 66/96] minor fix --- consensus/types/src/payload.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 38e58a6d74c..f9847f4f93e 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -608,7 +608,21 @@ impl ExecPayload for BlindedPayload { } fn deposit_receipts(&self) -> Result, Error> { - todo!() + match self { + BlindedPayload::Merge(_) => { + // Return an "empty" or "default" value for the Merge variant + Ok(Vec::new().into()) + } + BlindedPayload::Capella(_) => { + // Return an "empty" or "default" value for the Capella variant + Ok(Vec::new().into()) + } + BlindedPayload::Eip4844(_) => { + // Return an "empty" or "default" value for the EIP-4844 variant + Ok(Vec::new().into()) + } + BlindedPayload::Eip6110(ref inner) => Ok(inner.deposit_receipts()?.clone()), + } } } From d77f51b656338b06b363719e5b81387276acb27e Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Fri, 14 Apr 2023 19:51:21 +0200 Subject: [PATCH 67/96] clippy --- consensus/types/src/payload.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index f9847f4f93e..0d2760e9e6f 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -621,7 +621,8 @@ impl ExecPayload for BlindedPayload { // Return an "empty" or "default" value for the EIP-4844 variant Ok(Vec::new().into()) } - BlindedPayload::Eip6110(ref inner) => Ok(inner.deposit_receipts()?.clone()), + BlindedPayload::Eip6110(ref inner) => Ok(inner.deposit_receipts()?), + } } } From 96a4f36f79c6e2c7eca7208c318ea4404cbabf22 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Fri, 14 Apr 2023 19:52:51 +0200 Subject: [PATCH 68/96] fmt... --- consensus/types/src/payload.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 0d2760e9e6f..5846ce5fb44 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -622,7 +622,6 @@ impl ExecPayload for BlindedPayload { Ok(Vec::new().into()) } BlindedPayload::Eip6110(ref inner) => Ok(inner.deposit_receipts()?), - } } } From 042e66068fa3817657bdc271a5dcdbe437f68f69 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Sun, 16 Apr 2023 16:14:04 +0200 Subject: [PATCH 69/96] cleanup --- beacon_node/beacon_chain/src/test_utils.rs | 5 ++++- beacon_node/execution_layer/src/engine_api.rs | 2 +- .../src/test_utils/mock_builder.rs | 14 ++++++++------ .../lighthouse_network/src/types/topics.rs | 2 +- .../per_block_processing/process_operations.rs | 18 ++++++++---------- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 949310bdd7a..3bcd0d36743 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -447,12 +447,15 @@ where let eip4844_time = spec.eip4844_fork_epoch.map(|epoch| { HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); + let eip6110_time = spec.eip6110_fork_epoch.map(|epoch| { + HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() + }); let mock = MockExecutionLayer::new( self.runtime.task_executor.clone(), DEFAULT_TERMINAL_BLOCK, shanghai_time, eip4844_time, - None, + eip6110_time, None, Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec, diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 09be765fa4e..1b18108c262 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -302,7 +302,7 @@ impl TryFrom> for ExecutionBlockWithTransactions .collect(), deposit_receipts: Vec::from(block.deposit_receipts) .into_iter() - .map(|receipt| receipt.into()) + .map(|deposit_receipt| deposit_receipt.into()) .collect(), }) } diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index b5a500c3f3e..9ba63d7937c 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -442,17 +442,19 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { let json_payload = serde_json::to_string(&payload).map_err(convert_err)?; let mut message = match fork { - ForkName::Capella => BuilderBid::Capella(BuilderBidCapella { - header: serde_json::from_str(json_payload.as_str()).map_err(convert_err)?, - value: to_ssz_rs(&Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI))?, - public_key: self.builder_sk.public_key(), - }), + ForkName::Capella | ForkName::Eip4844 | ForkName::Eip6110 => { + BuilderBid::Capella(BuilderBidCapella { + header: serde_json::from_str(json_payload.as_str()).map_err(convert_err)?, + value: to_ssz_rs(&Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI))?, + public_key: self.builder_sk.public_key(), + }) + } ForkName::Merge => BuilderBid::Bellatrix(BuilderBidBellatrix { header: serde_json::from_str(json_payload.as_str()).map_err(convert_err)?, value: to_ssz_rs(&Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI))?, public_key: self.builder_sk.public_key(), }), - ForkName::Base | ForkName::Altair | ForkName::Eip4844 | ForkName::Eip6110 => { + ForkName::Base | ForkName::Altair => { return Err(BlindedBlockProviderError::Custom(format!( "Unsupported fork: {}", fork diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index a78f21af9cd..3dc0ebcdb0f 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -48,7 +48,7 @@ pub fn fork_core_topics(fork_name: &ForkName) -> Vec { ForkName::Merge => vec![], ForkName::Capella => CAPELLA_CORE_TOPICS.to_vec(), ForkName::Eip4844 => vec![], // TODO - ForkName::Eip6110 => vec![], // TODO + ForkName::Eip6110 => vec![], } } diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 3d32877b8c1..986a25eadb8 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -26,17 +26,15 @@ pub fn process_operations>( .unwrap_or_else(|_| state.eth1_data().deposit_count); if state.eth1_deposit_index() < eth1_deposit_index_limit { - assert_eq!( - block_body.deposits().len(), - std::cmp::min(max_deposits as usize, unprocessed_deposits_count as usize), - "Number of deposits in block does not match the minimum of the maximum number of deposits and the number of unprocessed deposits" - ); + if block_body.deposits().len() + != std::cmp::min(max_deposits as usize, unprocessed_deposits_count as usize) + { + return Err(BlockProcessingError::DepositReceiptError); + } } else { - assert_eq!( - block_body.deposits().len(), - 0, - "Number of deposits in block must be zero when all prior deposits are processed" - ); + if block_body.deposits().len() != 0 { + return Err(BlockProcessingError::DepositReceiptError); + } } process_proposer_slashings( From 40f5897dfa6dd527bf37df0c7b719059f268f536 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Sun, 16 Apr 2023 16:27:13 +0200 Subject: [PATCH 70/96] clippy --- .../src/per_block_processing/process_operations.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 986a25eadb8..e026a03944d 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -31,10 +31,8 @@ pub fn process_operations>( { return Err(BlockProcessingError::DepositReceiptError); } - } else { - if block_body.deposits().len() != 0 { - return Err(BlockProcessingError::DepositReceiptError); - } + } else if !block_body.deposits().is_empty() { + return Err(BlockProcessingError::DepositReceiptError); } process_proposer_slashings( From 9dee7181537a488067eec8bbe673bf93545d0088 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 21 Apr 2023 03:40:10 +1000 Subject: [PATCH 71/96] Remove unused blob endpoint and types (#4209) --- common/eth2/src/lib.rs | 26 -------------------------- common/eth2/src/types.rs | 31 ------------------------------- 2 files changed, 57 deletions(-) diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 5cd34faf2d9..9f7af399678 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -1443,32 +1443,6 @@ impl BeaconNodeHttpClient { self.get(path).await } - /// `GET v1/validator/blocks_and_blobs/{slot}` - pub async fn get_validator_blocks_and_blobs>( - &self, - slot: Slot, - randao_reveal: &SignatureBytes, - graffiti: Option<&Graffiti>, - ) -> Result>, Error> { - let mut path = self.eth_path(V1)?; - - path.path_segments_mut() - .map_err(|()| Error::InvalidUrl(self.server.clone()))? - .push("validator") - .push("blocks_and_blobs") - .push(&slot.to_string()); - - path.query_pairs_mut() - .append_pair("randao_reveal", &randao_reveal.to_string()); - - if let Some(graffiti) = graffiti { - path.query_pairs_mut() - .append_pair("graffiti", &graffiti.to_string()); - } - - self.get(path).await - } - /// `GET v2/validator/blinded_blocks/{slot}` pub async fn get_validator_blinded_blocks>( &self, diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 3f4273730d8..36f80afd1ed 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1218,37 +1218,6 @@ pub struct LivenessResponseData { pub is_live: bool, } -#[derive(PartialEq, Debug, Serialize, Deserialize)] -#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] -pub struct BlocksAndBlobs> { - pub block: BeaconBlock, - pub blobs: Vec>, - pub kzg_aggregate_proof: KzgProof, -} - -impl> ForkVersionDeserialize - for BlocksAndBlobs -{ - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( - value: serde_json::value::Value, - fork_name: ForkName, - ) -> Result { - #[derive(Deserialize)] - #[serde(bound = "T: EthSpec")] - struct Helper { - block: serde_json::Value, - blobs: Vec>, - kzg_aggregate_proof: KzgProof, - } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; - - Ok(Self { - block: BeaconBlock::deserialize_by_fork::<'de, D>(helper.block, fork_name)?, - blobs: helper.blobs, - kzg_aggregate_proof: helper.kzg_aggregate_proof, - }) - } -} #[derive(Debug, Serialize, Deserialize)] pub struct ForkChoice { pub justified_checkpoint: Checkpoint, From 2d083436c846df8e80cd97e433def843e2a83026 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 21 Apr 2023 03:58:49 +1000 Subject: [PATCH 72/96] Switch blob tx type to 0x03 (#4186) --- consensus/types/src/consts.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/types/src/consts.rs b/consensus/types/src/consts.rs index 8d296ad942f..6c60a985a5e 100644 --- a/consensus/types/src/consts.rs +++ b/consensus/types/src/consts.rs @@ -34,6 +34,6 @@ pub mod deneb { .expect("should initialize BLS_MODULUS"); pub static ref MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS: Epoch = Epoch::from(4096_u64); } - pub const BLOB_TX_TYPE: u8 = 5; + pub const BLOB_TX_TYPE: u8 = 3; pub const VERSIONED_HASH_VERSION_KZG: u8 = 1; } From a6335eb27e12ab3029fab47a6234f43e93f28aba Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 20 Apr 2023 14:00:25 -0400 Subject: [PATCH 73/96] bump ef tests version (#4217) --- testing/ef_tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index 38b863b9fb9..7eeb70c7728 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,4 +1,4 @@ -TESTS_TAG := v1.3.0-rc.5 +TESTS_TAG := v1.3.0 TESTS = general minimal mainnet TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) From e6b033aefda090468662dccf41cb15cceec8199e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 20 Apr 2023 18:23:59 -0400 Subject: [PATCH 74/96] update blob transaction (#4218) * update blob transaction * update blob transaction * rename in JSON deserializing --- beacon_node/execution_layer/src/lib.rs | 18 +++++++++--------- .../src/per_block_processing/deneb/deneb.rs | 10 +++++----- consensus/types/src/transaction.rs | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index a500bf9ee50..dc9522d88be 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -2080,8 +2080,8 @@ pub enum BlobTxConversionError { AccessListMissing, /// Missing the `max_fee_per_data_gas` field. MaxFeePerDataGasMissing, - /// Missing the `blob_versioned_hashes` field. - BlobVersionedHashesMissing, + /// Missing the `versioned_hashes` field. + VersionedHashesMissing, /// `y_parity` field was greater than one. InvalidYParity, /// There was an error converting the transaction to SSZ. @@ -2207,18 +2207,18 @@ fn ethers_tx_to_bytes( ) .map_err(BlobTxConversionError::FromStrRadix)?; - // blobVersionedHashes - let blob_versioned_hashes = other - .get("blobVersionedHashes") - .ok_or(BlobTxConversionError::BlobVersionedHashesMissing)? + // versionedHashes + let versioned_hashes = other + .get("versionedHashes") + .ok_or(BlobTxConversionError::VersionedHashesMissing)? .as_array() - .ok_or(BlobTxConversionError::BlobVersionedHashesMissing)? + .ok_or(BlobTxConversionError::VersionedHashesMissing)? .iter() .map(|versioned_hash| { let hash_bytes = eth2_serde_utils::hex::decode( versioned_hash .as_str() - .ok_or(BlobTxConversionError::BlobVersionedHashesMissing)?, + .ok_or(BlobTxConversionError::VersionedHashesMissing)?, ) .map_err(BlobTxConversionError::FromHex)?; if hash_bytes.len() != Hash256::ssz_fixed_len() { @@ -2239,7 +2239,7 @@ fn ethers_tx_to_bytes( data, access_list, max_fee_per_data_gas, - blob_versioned_hashes: VariableList::new(blob_versioned_hashes)?, + versioned_hashes: VariableList::new(versioned_hashes)?, }; // ******************** EcdsaSignature fields ******************** diff --git a/consensus/state_processing/src/per_block_processing/deneb/deneb.rs b/consensus/state_processing/src/per_block_processing/deneb/deneb.rs index 5935d6f2688..8ab213a4d0a 100644 --- a/consensus/state_processing/src/per_block_processing/deneb/deneb.rs +++ b/consensus/state_processing/src/per_block_processing/deneb/deneb.rs @@ -42,7 +42,7 @@ pub fn verify_kzg_commitments_against_transactions( .map(|tx_type| *tx_type == BLOB_TX_TYPE) .unwrap_or(false) }) - .map(|tx| tx_peek_blob_versioned_hashes::(tx)); + .map(|tx| tx_peek_versioned_hashes::(tx)); itertools::process_results(nested_iter, |iter| { let zipped_iter = iter @@ -76,7 +76,7 @@ pub fn verify_kzg_commitments_against_transactions( } /// Only transactions of type `BLOB_TX_TYPE` should be passed into this function. -fn tx_peek_blob_versioned_hashes( +fn tx_peek_versioned_hashes( opaque_tx: &Transaction, ) -> Result< impl IntoIterator> + '_, @@ -93,7 +93,7 @@ fn tx_peek_blob_versioned_hashes( let message_offset_usize = message_offset as usize; // field offset: 32 + 8 + 32 + 32 + 8 + 4 + 32 + 4 + 4 + 32 = 188 - let blob_versioned_hashes_offset = message_offset.safe_add(u32::from_ssz_bytes( + let versioned_hashes_offset = message_offset.safe_add(u32::from_ssz_bytes( opaque_tx .get(message_offset_usize.safe_add(188)?..message_offset_usize.safe_add(192)?) .ok_or(BlockProcessingError::BlobVersionHashIndexOutOfBounds { @@ -103,12 +103,12 @@ fn tx_peek_blob_versioned_hashes( )?)?; let num_hashes = tx_len - .safe_sub(blob_versioned_hashes_offset as usize)? + .safe_sub(versioned_hashes_offset as usize)? .safe_div(32)?; Ok((0..num_hashes).map(move |i| { let next_version_hash_index = - (blob_versioned_hashes_offset as usize).safe_add(i.safe_mul(32)?)?; + (versioned_hashes_offset as usize).safe_add(i.safe_mul(32)?)?; let bytes = opaque_tx .get(next_version_hash_index..next_version_hash_index.safe_add(32)?) .ok_or(BlockProcessingError::BlobVersionHashIndexOutOfBounds { diff --git a/consensus/types/src/transaction.rs b/consensus/types/src/transaction.rs index ee0af981b23..5a1c12ef159 100644 --- a/consensus/types/src/transaction.rs +++ b/consensus/types/src/transaction.rs @@ -1,13 +1,13 @@ use crate::{Hash256, Uint256, VersionedHash}; use ethereum_types::Address; use ssz_derive::{Decode, Encode}; -use ssz_types::typenum::U16777216; +use ssz_types::typenum::{U16777216, U4096}; use ssz_types::VariableList; pub type MaxCalldataSize = U16777216; pub type MaxAccessListSize = U16777216; -pub type MaxVersionedHashesListSize = U16777216; pub type MaxAccessListStorageKeys = U16777216; +pub type MaxVersionedHashesListSize = U4096; #[derive(Debug, Clone, PartialEq, Encode, Decode)] pub struct SignedBlobTransaction { @@ -27,7 +27,7 @@ pub struct BlobTransaction { pub data: VariableList, pub access_list: VariableList, pub max_fee_per_data_gas: Uint256, - pub blob_versioned_hashes: VariableList, + pub versioned_hashes: VariableList, } #[derive(Debug, Clone, PartialEq, Encode, Decode)] From 895bbd6c037fa88a553bfb79e55ae9e6236c5432 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Thu, 20 Apr 2023 15:26:20 -0700 Subject: [PATCH 75/96] Gossip conditions deneb (#4164) * Add all gossip conditions * Handle some gossip errors * Update beacon_node/beacon_chain/src/blob_verification.rs Co-authored-by: Divma <26765164+divagant-martian@users.noreply.github.com> * Add an ObservedBlobSidecars cache --------- Co-authored-by: Divma <26765164+divagant-martian@users.noreply.github.com> --- beacon_node/beacon_chain/src/beacon_chain.rs | 3 + .../beacon_chain/src/blob_verification.rs | 292 ++++++++++++------ .../beacon_chain/src/block_verification.rs | 2 +- beacon_node/beacon_chain/src/builder.rs | 2 + .../beacon_chain/src/canonical_head.rs | 7 + .../src/data_availability_checker.rs | 1 - beacon_node/beacon_chain/src/errors.rs | 3 + beacon_node/beacon_chain/src/lib.rs | 1 + .../src/observed_blob_sidecars.rs | 99 ++++++ .../beacon_processor/worker/gossip_methods.rs | 84 ++++- 10 files changed, 399 insertions(+), 95 deletions(-) create mode 100644 beacon_node/beacon_chain/src/observed_blob_sidecars.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 3b95b885835..ae085826c62 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -48,6 +48,7 @@ use crate::observed_aggregates::{ use crate::observed_attesters::{ ObservedAggregators, ObservedAttesters, ObservedSyncAggregators, ObservedSyncContributors, }; +use crate::observed_blob_sidecars::ObservedBlobSidecars; use crate::observed_block_producers::ObservedBlockProducers; use crate::observed_operations::{ObservationOutcome, ObservedOperations}; use crate::persisted_beacon_chain::{PersistedBeaconChain, DUMMY_CANONICAL_HEAD_BLOCK_ROOT}; @@ -401,6 +402,8 @@ pub struct BeaconChain { pub(crate) observed_sync_aggregators: RwLock>, /// Maintains a record of which validators have proposed blocks for each slot. pub(crate) observed_block_producers: RwLock>, + /// Maintains a record of blob sidecars seen over the gossip network. + pub(crate) observed_blob_sidecars: RwLock>, /// Maintains a record of which validators have submitted voluntary exits. pub(crate) observed_voluntary_exits: Mutex>, /// Maintains a record of which validators we've seen proposer slashings for. diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index bab8646e914..24524bddfaa 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -1,5 +1,6 @@ use derivative::Derivative; use slot_clock::SlotClock; +use state_processing::state_advance::partial_state_advance; use std::sync::Arc; use crate::beacon_chain::{ @@ -12,9 +13,11 @@ use crate::data_availability_checker::{ use crate::kzg_utils::{validate_blob, validate_blobs}; use crate::BeaconChainError; use kzg::Kzg; +use std::borrow::Cow; use types::{ - BeaconBlockRef, BeaconStateError, BlobSidecar, BlobSidecarList, Epoch, EthSpec, Hash256, - KzgCommitment, SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlobSidecar, Slot, + BeaconBlockRef, BeaconState, BeaconStateError, BlobSidecar, BlobSidecarList, ChainSpec, + CloneConfig, Epoch, EthSpec, Hash256, KzgCommitment, RelativeEpoch, SignedBeaconBlock, + SignedBeaconBlockHeader, SignedBlobSidecar, Slot, }; #[derive(Debug)] @@ -30,31 +33,6 @@ pub enum BlobError { latest_permissible_slot: Slot, }, - /// The blob sidecar has a different slot than the block. - /// - /// ## Peer scoring - /// - /// Assuming the local clock is correct, the peer has sent an invalid message. - SlotMismatch { - blob_slot: Slot, - block_slot: Slot, - }, - - /// No kzg ccommitment associated with blob sidecar. - KzgCommitmentMissing, - - /// No transactions in block - TransactionsMissing, - - /// Blob transactions in the block do not correspond to the kzg commitments. - TransactionCommitmentMismatch, - - TrustedSetupNotInitialized, - - InvalidKzgProof, - - KzgError(kzg::Error), - /// There was an error whilst processing the sync contribution. It is not known if it is valid or invalid. /// /// ## Peer scoring @@ -62,28 +40,20 @@ pub enum BlobError { /// We were unable to process this sync committee message due to an internal error. It's unclear if the /// sync committee message is valid. BeaconChainError(BeaconChainError), - /// No blobs for the specified block where we would expect blobs. - UnavailableBlobs, - /// Blobs provided for a pre-Deneb fork. - InconsistentFork, - /// The `blobs_sidecar.message.beacon_block_root` block is unknown. + /// The `BlobSidecar` was gossiped over an incorrect subnet. /// /// ## Peer scoring /// - /// The blob points to a block we have not yet imported. The blob cannot be imported - /// into fork choice yet - UnknownHeadBlock { - beacon_block_root: Hash256, - }, - - /// The `BlobSidecar` was gossiped over an incorrect subnet. - InvalidSubnet { - expected: u64, - received: u64, - }, + /// The blob is invalid or the peer is faulty. + InvalidSubnet { expected: u64, received: u64 }, /// The sidecar corresponds to a slot older than the finalized head slot. + /// + /// ## Peer scoring + /// + /// It's unclear if this blob is valid, but this blob is for a finalized slot and is + /// therefore useless to us. PastFinalizedSlot { blob_slot: Slot, finalized_slot: Slot, @@ -91,21 +61,19 @@ pub enum BlobError { /// The proposer index specified in the sidecar does not match the locally computed /// proposer index. - ProposerIndexMismatch { - sidecar: usize, - local: usize, - }, + /// + /// ## Peer scoring + /// + /// The blob is invalid and the peer is faulty. + ProposerIndexMismatch { sidecar: usize, local: usize }, + /// The proposal signature in invalid. + /// + /// ## Peer scoring + /// + /// The blob is invalid and the peer is faulty. ProposerSignatureInvalid, - /// A sidecar with same slot, beacon_block_root and proposer_index but different blob is received for - /// the same blob index. - RepeatSidecar { - proposer: usize, - slot: Slot, - blob_index: usize, - }, - /// The proposal_index corresponding to blob.beacon_block_root is not known. /// /// ## Peer scoring @@ -113,7 +81,34 @@ pub enum BlobError { /// The block is invalid and the peer is faulty. UnknownValidator(u64), - BlobCacheError(AvailabilityCheckError), + /// The provided blob is not from a later slot than its parent. + /// + /// ## Peer scoring + /// + /// The blob is invalid and the peer is faulty. + BlobIsNotLaterThanParent { blob_slot: Slot, parent_slot: Slot }, + + /// The provided blob's parent block is unknown. + /// + /// ## Peer scoring + /// + /// We cannot process the blob without validating its parent, the peer isn't necessarily faulty. + BlobParentUnknown { + blob_root: Hash256, + blob_parent_root: Hash256, + }, + + /// A blob has already been seen for the given `(sidecar.block_root, sidecar.index)` tuple + /// over gossip or no gossip sources. + /// + /// ## Peer scoring + /// + /// The peer isn't faulty, but we do not forward it over gossip. + RepeatBlob { + proposer: u64, + slot: Slot, + index: u64, + }, } impl From for BlobError { @@ -149,6 +144,8 @@ pub fn validate_blob_sidecar_for_gossip( let blob_slot = signed_blob_sidecar.message.slot; let blob_index = signed_blob_sidecar.message.index; let block_root = signed_blob_sidecar.message.block_root; + let block_parent_root = signed_blob_sidecar.message.block_parent_root; + let blob_proposer_index = signed_blob_sidecar.message.proposer_index; // Verify that the blob_sidecar was received on the correct subnet. if blob_index != subnet { @@ -170,8 +167,6 @@ pub fn validate_blob_sidecar_for_gossip( }); } - // TODO(pawan): Verify not from a past slot? - // Verify that the sidecar slot is greater than the latest finalized slot let latest_finalized_slot = chain .head() @@ -185,26 +180,93 @@ pub fn validate_blob_sidecar_for_gossip( }); } - // TODO(pawan): should we verify locally that the parent root is correct - // or just use whatever the proposer gives us? + // Verify that this is the first blob sidecar received for the (sidecar.block_root, sidecar.index) tuple + if chain + .observed_blob_sidecars + .read() + .is_known(&signed_blob_sidecar.message) + .map_err(|e| BlobError::BeaconChainError(e.into()))? + { + return Err(BlobError::RepeatBlob { + proposer: blob_proposer_index, + slot: blob_slot, + index: blob_index, + }); + } + + // We have already verified that the blob is past finalization, so we can + // just check fork choice for the block's parent. + if let Some(parent_block) = chain + .canonical_head + .fork_choice_read_lock() + .get_block(&block_parent_root) + { + if parent_block.slot >= blob_slot { + return Err(BlobError::BlobIsNotLaterThanParent { + blob_slot, + parent_slot: parent_block.slot, + }); + } + } else { + return Err(BlobError::BlobParentUnknown { + blob_root: block_root, + blob_parent_root: block_parent_root, + }); + } + + // Note: The spec checks the signature directly against `blob_sidecar.message.proposer_index` + // before checking that the provided proposer index is valid w.r.t the current shuffling. + // + // However, we check that the proposer_index matches against the shuffling first to avoid + // signature verification against an invalid proposer_index. let proposer_shuffling_root = signed_blob_sidecar.message.block_parent_root; - let (proposer_index, fork) = match chain + let proposer_opt = chain .beacon_proposer_cache .lock() - .get_slot::(proposer_shuffling_root, blob_slot) - { - Some(proposer) => (proposer.index, proposer.fork), - None => { - let state = &chain.canonical_head.cached_head().snapshot.beacon_state; + .get_slot::(proposer_shuffling_root, blob_slot); + + let (proposer_index, fork) = if let Some(proposer) = proposer_opt { + (proposer.index, proposer.fork) + } else { + // The cached head state is in the same epoch as the blob or the state has already been + // advanced to the blob's epoch + let snapshot = &chain.canonical_head.cached_head().snapshot; + if snapshot.beacon_state.current_epoch() == blob_slot.epoch(T::EthSpec::slots_per_epoch()) { ( - state.get_beacon_proposer_index(blob_slot, &chain.spec)?, - state.fork(), + snapshot + .beacon_state + .get_beacon_proposer_index(blob_slot, &chain.spec)?, + snapshot.beacon_state.fork(), ) } + // Need to advance the state to get the proposer index + else { + // The state produced is only valid for determining proposer/attester shuffling indices. + let mut cloned_state = snapshot.clone_with(CloneConfig::committee_caches_only()); + let state = cheap_state_advance_to_obtain_committees( + &mut cloned_state.beacon_state, + None, + blob_slot, + &chain.spec, + )?; + + let proposers = state.get_beacon_proposer_indices(&chain.spec)?; + let proposer_index = *proposers + .get(blob_slot.as_usize() % T::EthSpec::slots_per_epoch() as usize) + .ok_or_else(|| BeaconChainError::NoProposerForSlot(blob_slot))?; + + // Prime the proposer shuffling cache with the newly-learned value. + chain.beacon_proposer_cache.lock().insert( + blob_slot.epoch(T::EthSpec::slots_per_epoch()), + proposer_shuffling_root, + proposers, + state.fork(), + )?; + (proposer_index, state.fork()) + } }; - let blob_proposer_index = signed_blob_sidecar.message.proposer_index; if proposer_index != blob_proposer_index as usize { return Err(BlobError::ProposerIndexMismatch { sidecar: blob_proposer_index as usize, @@ -212,6 +274,7 @@ pub fn validate_blob_sidecar_for_gossip( }); } + // Signature verification let signature_is_valid = { let pubkey_cache = chain .validator_pubkey_cache @@ -236,25 +299,27 @@ pub fn validate_blob_sidecar_for_gossip( return Err(BlobError::ProposerSignatureInvalid); } - // TODO(pawan): kzg validations. - - // TODO(pawan): Check if other blobs for the same proposer index and blob index have been - // received and drop if required. - - // Verify if the corresponding block for this blob has been received. - // Note: this should be the last gossip check so that we can forward the blob - // over the gossip network even if we haven't received the corresponding block yet - // as all other validations have passed. - let block_opt = chain - .canonical_head - .fork_choice_read_lock() - .get_block(&block_root) - .or_else(|| chain.early_attester_cache.get_proto_block(block_root)); // TODO(pawan): should we be checking this cache? - - // TODO(pawan): this may be redundant with the new `AvailabilityProcessingStatus::PendingBlock variant` - if block_opt.is_none() { - return Err(BlobError::UnknownHeadBlock { - beacon_block_root: block_root, + // Now the signature is valid, store the proposal so we don't accept another blob sidecar + // with the same `BlobIdentifier`. + // It's important to double-check that the proposer still hasn't been observed so we don't + // have a race-condition when verifying two blocks simultaneously. + // + // Note: If this BlobSidecar goes on to fail full verification, we do not evict it from the seen_cache + // as alternate blob_sidecars for the same identifier can still be retrieved + // over rpc. Evicting them from this cache would allow faster propagation over gossip. So we allow + // retreieval of potentially valid blocks over rpc, but try to punish the proposer for signing + // invalid messages. Issue for more background + // https://github.com/ethereum/consensus-specs/issues/3261 + if chain + .observed_blob_sidecars + .write() + .observe_sidecar(&signed_blob_sidecar.message) + .map_err(|e| BlobError::BeaconChainError(e.into()))? + { + return Err(BlobError::RepeatBlob { + proposer: proposer_index as u64, + slot: blob_slot, + index: blob_index, }); } @@ -263,6 +328,57 @@ pub fn validate_blob_sidecar_for_gossip( }) } +/// Performs a cheap (time-efficient) state advancement so the committees and proposer shuffling for +/// `slot` can be obtained from `state`. +/// +/// The state advancement is "cheap" since it does not generate state roots. As a result, the +/// returned state might be holistically invalid but the committees/proposers will be correct (since +/// they do not rely upon state roots). +/// +/// If the given `state` can already serve the `slot`, the committees will be built on the `state` +/// and `Cow::Borrowed(state)` will be returned. Otherwise, the state will be cloned, cheaply +/// advanced and then returned as a `Cow::Owned`. The end result is that the given `state` is never +/// mutated to be invalid (in fact, it is never changed beyond a simple committee cache build). +/// +/// Note: This is a copy of the `block_verification::cheap_state_advance_to_obtain_committees` to return +/// a BlobError error type instead. +/// TODO(pawan): try to unify the 2 functions. +fn cheap_state_advance_to_obtain_committees<'a, E: EthSpec>( + state: &'a mut BeaconState, + state_root_opt: Option, + blob_slot: Slot, + spec: &ChainSpec, +) -> Result>, BlobError> { + let block_epoch = blob_slot.epoch(E::slots_per_epoch()); + + if state.current_epoch() == block_epoch { + // Build both the current and previous epoch caches, as the previous epoch caches are + // useful for verifying attestations in blocks from the current epoch. + state.build_committee_cache(RelativeEpoch::Previous, spec)?; + state.build_committee_cache(RelativeEpoch::Current, spec)?; + + Ok(Cow::Borrowed(state)) + } else if state.slot() > blob_slot { + Err(BlobError::BlobIsNotLaterThanParent { + blob_slot, + parent_slot: state.slot(), + }) + } else { + let mut state = state.clone_with(CloneConfig::committee_caches_only()); + let target_slot = block_epoch.start_slot(E::slots_per_epoch()); + + // Advance the state into the same epoch as the block. Use the "partial" method since state + // roots are not important for proposer/attester shuffling. + partial_state_advance(&mut state, state_root_opt, target_slot, spec) + .map_err(|e| BlobError::BeaconChainError(BeaconChainError::from(e)))?; + + state.build_committee_cache(RelativeEpoch::Previous, spec)?; + state.build_committee_cache(RelativeEpoch::Current, spec)?; + + Ok(Cow::Owned(state)) + } +} + /// Wrapper over a `BlobSidecar` for which we have completed kzg verification. /// i.e. `verify_blob_kzg_proof(blob, commitment, proof) == true`. #[derive(Debug, Derivative, Clone)] diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 36cc7233193..94316c0d309 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -243,7 +243,7 @@ pub enum BlockError { /// /// The block is invalid and the peer is faulty. InvalidSignature, - /// The provided block is from an later slot than its parent. + /// The provided block is not from a later slot than its parent. /// /// ## Peer scoring /// diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 7620a588d68..d5f1146cdf0 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -813,6 +813,8 @@ where // TODO: allow for persisting and loading the pool from disk. observed_block_producers: <_>::default(), // TODO: allow for persisting and loading the pool from disk. + observed_blob_sidecars: <_>::default(), + // TODO: allow for persisting and loading the pool from disk. observed_voluntary_exits: <_>::default(), observed_proposer_slashings: <_>::default(), observed_attester_slashings: <_>::default(), diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 685b8f65594..e3adca9ca7a 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -952,6 +952,13 @@ impl BeaconChain { .start_slot(T::EthSpec::slots_per_epoch()), ); + self.observed_blob_sidecars.write().prune( + new_view + .finalized_checkpoint + .epoch + .start_slot(T::EthSpec::slots_per_epoch()), + ); + self.snapshot_cache .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) .map(|mut snapshot_cache| { diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 4c191695b20..d9446e1ec0f 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -65,7 +65,6 @@ pub struct DataAvailabilityChecker { /// The blobs are all gossip and kzg verified. /// The block has completed all verifications except the availability check. struct ReceivedComponents { - /// We use a `BTreeMap` here to maintain the order of `BlobSidecar`s based on index. verified_blobs: FixedVector>, T::MaxBlobsPerBlock>, executed_block: Option>, } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 9baa638f450..9d5485df9ed 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -8,6 +8,7 @@ use crate::migrate::PruningError; use crate::naive_aggregation_pool::Error as NaiveAggregationError; use crate::observed_aggregates::Error as ObservedAttestationsError; use crate::observed_attesters::Error as ObservedAttestersError; +use crate::observed_blob_sidecars::Error as ObservedBlobSidecarsError; use crate::observed_block_producers::Error as ObservedBlockProducersError; use execution_layer::PayloadStatus; use fork_choice::ExecutionStatus; @@ -101,6 +102,7 @@ pub enum BeaconChainError { ObservedAttestationsError(ObservedAttestationsError), ObservedAttestersError(ObservedAttestersError), ObservedBlockProducersError(ObservedBlockProducersError), + ObservedBlobSidecarsError(ObservedBlobSidecarsError), AttesterCacheError(AttesterCacheError), PruningError(PruningError), ArithError(ArithError), @@ -228,6 +230,7 @@ easy_from_to!(NaiveAggregationError, BeaconChainError); easy_from_to!(ObservedAttestationsError, BeaconChainError); easy_from_to!(ObservedAttestersError, BeaconChainError); easy_from_to!(ObservedBlockProducersError, BeaconChainError); +easy_from_to!(ObservedBlobSidecarsError, BeaconChainError); easy_from_to!(AttesterCacheError, BeaconChainError); easy_from_to!(BlockSignatureVerifierError, BeaconChainError); easy_from_to!(PruningError, BeaconChainError); diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index c76ba752a85..450c48f293f 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -36,6 +36,7 @@ pub mod migrate; mod naive_aggregation_pool; mod observed_aggregates; mod observed_attesters; +mod observed_blob_sidecars; mod observed_block_producers; pub mod observed_operations; pub mod otb_verification_service; diff --git a/beacon_node/beacon_chain/src/observed_blob_sidecars.rs b/beacon_node/beacon_chain/src/observed_blob_sidecars.rs new file mode 100644 index 00000000000..1577d5951ca --- /dev/null +++ b/beacon_node/beacon_chain/src/observed_blob_sidecars.rs @@ -0,0 +1,99 @@ +//! Provides the `ObservedBlobSidecars` struct which allows for rejecting `BlobSidecar`s +//! that we have already seen over the gossip network. +//! Only `BlobSidecar`s that have completed proposer signature verification can be added +//! to this cache to reduce DoS risks. + +use std::collections::{HashMap, HashSet}; +use std::marker::PhantomData; +use std::sync::Arc; +use types::{BlobSidecar, EthSpec, Hash256, Slot}; + +#[derive(Debug, PartialEq)] +pub enum Error { + /// The slot of the provided `BlobSidecar` is prior to finalization and should not have been provided + /// to this function. This is an internal error. + FinalizedBlob { slot: Slot, finalized_slot: Slot }, + /// The blob sidecar contains an invalid blob index, the blob sidecar is invalid. + /// Note: The invalid blob should have been caught and flagged as an error much before reaching + /// here. + InvalidBlobIndex(u64), +} + +/// Maintains a cache of seen `BlobSidecar`s that are received over gossip +/// and have been gossip verified. +/// +/// The cache supports pruning based upon the finalized epoch. It does not automatically prune, you +/// must call `Self::prune` manually. +/// +/// Note: To prevent DoS attacks, this cache must include only items that have received some DoS resistance +/// like checking the proposer signature. +pub struct ObservedBlobSidecars { + finalized_slot: Slot, + /// Stores all received blob indices for a given `(Root, Slot)` tuple. + items: HashMap<(Hash256, Slot), HashSet>, + _phantom: PhantomData, +} + +impl Default for ObservedBlobSidecars { + /// Instantiates `Self` with `finalized_slot == 0`. + fn default() -> Self { + Self { + finalized_slot: Slot::new(0), + items: HashMap::new(), + _phantom: PhantomData, + } + } +} + +impl ObservedBlobSidecars { + /// Observe the `blob_sidecar` at (`blob_sidecar.block_root, blob_sidecar.slot`). + /// This will update `self` so future calls to it indicate that this `blob_sidecar` is known. + /// + /// The supplied `blob_sidecar` **MUST** have completed proposer signature verification. + pub fn observe_sidecar(&mut self, blob_sidecar: &Arc>) -> Result { + self.sanitize_blob_sidecar(blob_sidecar)?; + + let did_not_exist = self + .items + .entry((blob_sidecar.block_root, blob_sidecar.slot)) + .or_insert_with(|| HashSet::with_capacity(T::max_blobs_per_block())) + .insert(blob_sidecar.index); + + Ok(!did_not_exist) + } + + /// Returns `true` if the `blob_sidecar` has already been observed in the cache within the prune window. + pub fn is_known(&self, blob_sidecar: &Arc>) -> Result { + self.sanitize_blob_sidecar(blob_sidecar)?; + let is_known = self + .items + .get(&(blob_sidecar.block_root, blob_sidecar.slot)) + .map_or(false, |set| set.contains(&blob_sidecar.index)); + Ok(is_known) + } + + fn sanitize_blob_sidecar(&self, blob_sidecar: &Arc>) -> Result<(), Error> { + if blob_sidecar.index >= T::max_blobs_per_block() as u64 { + return Err(Error::InvalidBlobIndex(blob_sidecar.index)); + } + let finalized_slot = self.finalized_slot; + if finalized_slot > 0 && blob_sidecar.slot <= finalized_slot { + return Err(Error::FinalizedBlob { + slot: blob_sidecar.slot, + finalized_slot, + }); + } + + Ok(()) + } + + /// Prune all values earlier than the given slot. + pub fn prune(&mut self, finalized_slot: Slot) { + if finalized_slot == 0 { + return; + } + + self.finalized_slot = finalized_slot; + self.items.retain(|k, _| k.1 > finalized_slot); + } +} diff --git a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs index 69e167b4d03..c3298d8700c 100644 --- a/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/gossip_methods.rs @@ -1,6 +1,6 @@ use crate::{metrics, service::NetworkMessage, sync::SyncMessage}; -use beacon_chain::blob_verification::{AsBlock, BlockWrapper, GossipVerifiedBlob}; +use beacon_chain::blob_verification::{AsBlock, BlobError, BlockWrapper, GossipVerifiedBlob}; use beacon_chain::store::Error; use beacon_chain::{ attestation_verification::{self, Error as AttnError, VerifiedAttestation}, @@ -651,24 +651,98 @@ impl Worker { #[allow(clippy::too_many_arguments)] pub async fn process_gossip_blob( self, - _message_id: MessageId, + message_id: MessageId, peer_id: PeerId, _peer_client: Client, blob_index: u64, signed_blob: SignedBlobSidecar, _seen_duration: Duration, ) { + let slot = signed_blob.message.slot; + let root = signed_blob.message.block_root; + let index = signed_blob.message.index; match self .chain .verify_blob_sidecar_for_gossip(signed_blob, blob_index) { Ok(gossip_verified_blob) => { + debug!( + self.log, + "Successfully verified gossip blob"; + "slot" => %slot, + "root" => %root, + "index" => %index + ); + self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept); self.process_gossip_verified_blob(peer_id, gossip_verified_blob, _seen_duration) .await } - Err(_) => { - // TODO(pawan): handle all blob errors for peer scoring - todo!() + Err(err) => { + match err { + BlobError::BlobParentUnknown { + blob_root, + blob_parent_root, + } => { + debug!( + self.log, + "Unknown parent hash for blob"; + "action" => "requesting parent", + "blob_root" => %blob_root, + "parent_root" => %blob_parent_root + ); + // TODO: send blob to reprocessing queue and queue a sync request for the blob. + todo!(); + } + BlobError::ProposerSignatureInvalid + | BlobError::UnknownValidator(_) + | BlobError::ProposerIndexMismatch { .. } + | BlobError::BlobIsNotLaterThanParent { .. } + | BlobError::InvalidSubnet { .. } => { + warn!( + self.log, + "Could not verify blob sidecar for gossip. Rejecting the blob sidecar"; + "error" => ?err, + "slot" => %slot, + "root" => %root, + "index" => %index + ); + // Prevent recurring behaviour by penalizing the peer slightly. + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "gossip_blob_low", + ); + self.propagate_validation_result( + message_id, + peer_id, + MessageAcceptance::Reject, + ); + } + BlobError::FutureSlot { .. } + | BlobError::BeaconChainError(_) + | BlobError::RepeatBlob { .. } + | BlobError::PastFinalizedSlot { .. } => { + warn!( + self.log, + "Could not verify blob sidecar for gossip. Ignoring the blob sidecar"; + "error" => ?err, + "slot" => %slot, + "root" => %root, + "index" => %index + ); + // Prevent recurring behaviour by penalizing the peer slightly. + self.gossip_penalize_peer( + peer_id, + PeerAction::HighToleranceError, + "gossip_blob_high", + ); + self.propagate_validation_result( + message_id, + peer_id, + MessageAcceptance::Ignore, + ); + } + } } } } From 7a36d004e48a1a94196f916101a7a419259decbe Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Sat, 22 Apr 2023 06:21:09 -0700 Subject: [PATCH 76/96] Subscribe blob topics (#4224) --- .../lighthouse_network/src/service/mod.rs | 2 +- .../lighthouse_network/src/types/topics.rs | 30 ++++++++++++++----- beacon_node/network/src/service.rs | 4 ++- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 0b96f2e625f..7aab9f7d59a 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -564,7 +564,7 @@ impl Network { } // Subscribe to core topics for the new fork - for kind in fork_core_topics(&new_fork) { + for kind in fork_core_topics::(&new_fork) { let topic = GossipTopic::new(kind, GossipEncoding::default(), new_fork_digest); self.subscribe(topic); } diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 3a35c070250..db8894035c8 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -1,7 +1,7 @@ use libp2p::gossipsub::{IdentTopic as Topic, TopicHash}; use serde_derive::{Deserialize, Serialize}; use strum::AsRefStr; -use types::{ForkName, SubnetId, SyncSubnetId}; +use types::{EthSpec, ForkName, SubnetId, SyncSubnetId}; use crate::Subnet; @@ -40,23 +40,34 @@ pub const LIGHT_CLIENT_GOSSIP_TOPICS: [GossipKind; 2] = [ GossipKind::LightClientOptimisticUpdate, ]; +pub const DENEB_CORE_TOPICS: [GossipKind; 0] = []; + /// Returns the core topics associated with each fork that are new to the previous fork -pub fn fork_core_topics(fork_name: &ForkName) -> Vec { +pub fn fork_core_topics(fork_name: &ForkName) -> Vec { match fork_name { ForkName::Base => BASE_CORE_TOPICS.to_vec(), ForkName::Altair => ALTAIR_CORE_TOPICS.to_vec(), ForkName::Merge => vec![], ForkName::Capella => CAPELLA_CORE_TOPICS.to_vec(), - ForkName::Deneb => vec![], // TODO + ForkName::Deneb => { + // All of deneb blob topics are core topics + let mut deneb_blob_topics = Vec::new(); + for i in 0..T::max_blobs_per_block() { + deneb_blob_topics.push(GossipKind::BlobSidecar(i as u64)); + } + let mut deneb_topics = DENEB_CORE_TOPICS.to_vec(); + deneb_topics.append(&mut deneb_blob_topics); + deneb_topics + } } } /// Returns all the topics that we need to subscribe to for a given fork /// including topics from older forks and new topics for the current fork. -pub fn core_topics_to_subscribe(mut current_fork: ForkName) -> Vec { - let mut topics = fork_core_topics(¤t_fork); +pub fn core_topics_to_subscribe(mut current_fork: ForkName) -> Vec { + let mut topics = fork_core_topics::(¤t_fork); while let Some(previous_fork) = current_fork.previous_fork() { - let previous_fork_topics = fork_core_topics(&previous_fork); + let previous_fork_topics = fork_core_topics::(&previous_fork); topics.extend(previous_fork_topics); current_fork = previous_fork; } @@ -292,6 +303,8 @@ fn subnet_topic_index(topic: &str) -> Option { #[cfg(test)] mod tests { + use types::MainnetEthSpec; + use super::GossipKind::*; use super::*; @@ -420,12 +433,15 @@ mod tests { #[test] fn test_core_topics_to_subscribe() { + type E = MainnetEthSpec; let mut all_topics = Vec::new(); + let mut deneb_core_topics = fork_core_topics::(&ForkName::Deneb); + all_topics.append(&mut deneb_core_topics); all_topics.extend(CAPELLA_CORE_TOPICS); all_topics.extend(ALTAIR_CORE_TOPICS); all_topics.extend(BASE_CORE_TOPICS); let latest_fork = *ForkName::list_all().last().unwrap(); - assert_eq!(core_topics_to_subscribe(latest_fork), all_topics); + assert_eq!(core_topics_to_subscribe::(latest_fork), all_topics); } } diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index e443a51dce4..265a41189cb 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -690,7 +690,9 @@ impl NetworkService { } let mut subscribed_topics: Vec = vec![]; - for topic_kind in core_topics_to_subscribe(self.fork_context.current_fork()) { + for topic_kind in + core_topics_to_subscribe::(self.fork_context.current_fork()) + { for fork_digest in self.required_gossip_fork_digests() { let topic = GossipTopic::new( topic_kind.clone(), From c96d79d3f6de0ed54d953fb2917d35b2e1776b07 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Mon, 24 Apr 2023 11:41:57 +0200 Subject: [PATCH 77/96] add `upgrade_to_eip6110` --- consensus/state_processing/src/genesis.rs | 102 +++++++++++++++++----- 1 file changed, 79 insertions(+), 23 deletions(-) diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 10b9577400b..bd592c9e082 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -3,6 +3,10 @@ use super::per_block_processing::{ }; use crate::common::DepositDataTree; use crate::per_block_processing::UNSET_DEPOSIT_RECEIPTS_START_INDEX; +use crate::upgrade::{ + upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_eip4844, + upgrade_to_eip6110, +}; use safe_arith::{ArithError, SafeArith}; use tree_hash::TreeHash; use types::DEPOSIT_TREE_DEPTH; @@ -51,29 +55,81 @@ pub fn initialize_beacon_state_from_eth1( // Add deposit_receipts_start_index field with the value UNSET_DEPOSIT_RECEIPTS_START_INDEX *state.deposit_receipts_start_index_mut()? = UNSET_DEPOSIT_RECEIPTS_START_INDEX; - // Initialize the execution payload header - if let Some(header) = execution_payload_header { - match state.latest_execution_payload_header_mut()? { - ExecutionPayloadHeaderRefMut::Merge(header_mut) => { - if let ExecutionPayloadHeader::Merge(header) = header { - *header_mut = header; - } - } - ExecutionPayloadHeaderRefMut::Capella(header_mut) => { - if let ExecutionPayloadHeader::Capella(header) = header { - *header_mut = header; - } - } - ExecutionPayloadHeaderRefMut::Eip4844(header_mut) => { - if let ExecutionPayloadHeader::Eip4844(header) = header { - *header_mut = header; - } - } - ExecutionPayloadHeaderRefMut::Eip6110(header_mut) => { - if let ExecutionPayloadHeader::Eip6110(header) = header { - *header_mut = header; - } - } + if spec + .altair_fork_epoch + .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) + { + upgrade_to_altair(&mut state, spec)?; + + state.fork_mut().previous_version = spec.altair_fork_version; + } + + // Similarly, perform an upgrade to the merge if configured from genesis. + if spec + .bellatrix_fork_epoch + .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) + { + // this will set state.latest_execution_payload_header = ExecutionPayloadHeaderMerge::default() + upgrade_to_bellatrix(&mut state, spec)?; + + // Remove intermediate Altair fork from `state.fork`. + state.fork_mut().previous_version = spec.bellatrix_fork_version; + + // Override latest execution payload header. + // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/bellatrix/beacon-chain.md#testing + if let Some(ExecutionPayloadHeader::Merge(ref header)) = execution_payload_header { + *state.latest_execution_payload_header_merge_mut()? = header.clone(); + } + } + + // Upgrade to capella if configured from genesis + if spec + .capella_fork_epoch + .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) + { + upgrade_to_capella(&mut state, spec)?; + + // Remove intermediate Bellatrix fork from `state.fork`. + state.fork_mut().previous_version = spec.capella_fork_version; + + // Override latest execution payload header. + // See https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#testing + if let Some(ExecutionPayloadHeader::Capella(ref header)) = execution_payload_header { + *state.latest_execution_payload_header_capella_mut()? = header.clone(); + } + } + + // Upgrade to eip4844 if configured from genesis + if spec + .eip4844_fork_epoch + .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) + { + upgrade_to_eip4844(&mut state, spec)?; + + // Remove intermediate Capella fork from `state.fork`. + state.fork_mut().previous_version = spec.eip4844_fork_version; + + // Override latest execution payload header. + // See https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#testing + if let Some(ExecutionPayloadHeader::Eip4844(ref header)) = execution_payload_header { + *state.latest_execution_payload_header_eip4844_mut()? = header.clone(); + } + } + + // Upgrade to eip6110 if configured from genesis + if spec + .eip6110_fork_epoch + .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) + { + upgrade_to_eip6110(&mut state, spec)?; + + // Remove intermediate Eip4844 fork from `state.fork`. + state.fork_mut().previous_version = spec.eip6110_fork_version; + + // Override latest execution payload header. + // See https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip6110/beacon-chain.md#testing + if let Some(ExecutionPayloadHeader::Eip6110(header)) = execution_payload_header { + *state.latest_execution_payload_header_eip6110_mut()? = header; } } From cbe2e479312f92ec522ada00568f5a1a7507274e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 24 Apr 2023 09:03:23 -0400 Subject: [PATCH 78/96] update blobs by range protocol name (#4229) --- beacon_node/lighthouse_network/src/rpc/protocol.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index d627ae9f6e6..ed3b5a779fe 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -190,7 +190,7 @@ pub enum Protocol { #[strum(serialize = "beacon_blocks_by_root")] BlocksByRoot, /// The `BlobsByRange` protocol name. - #[strum(serialize = "blobs_sidecars_by_range")] + #[strum(serialize = "blob_sidecars_by_range")] BlobsByRange, /// The `BlobsByRoot` protocol name. #[strum(serialize = "blob_sidecars_by_root")] From a632969695942d59f20c5f7ce915a9d190fc1114 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Wed, 26 Apr 2023 07:44:58 -0700 Subject: [PATCH 79/96] Gossip verification cleanup (#4219) * Add ObservedBlobSidecar tests * Add logging for tricky verification cases * Update beacon_node/beacon_chain/src/blob_verification.rs --------- Co-authored-by: realbigsean --- .../beacon_chain/src/blob_verification.rs | 24 +- .../src/observed_blob_sidecars.rs | 290 ++++++++++++++++++ 2 files changed, 310 insertions(+), 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 24524bddfaa..1e561cf34b8 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -13,6 +13,7 @@ use crate::data_availability_checker::{ use crate::kzg_utils::{validate_blob, validate_blobs}; use crate::BeaconChainError; use kzg::Kzg; +use slog::{debug, warn}; use std::borrow::Cow; use types::{ BeaconBlockRef, BeaconState, BeaconStateError, BlobSidecar, BlobSidecarList, ChainSpec, @@ -214,10 +215,7 @@ pub fn validate_blob_sidecar_for_gossip( }); } - // Note: The spec checks the signature directly against `blob_sidecar.message.proposer_index` - // before checking that the provided proposer index is valid w.r.t the current shuffling. - // - // However, we check that the proposer_index matches against the shuffling first to avoid + // Note: We check that the proposer_index matches against the shuffling first to avoid // signature verification against an invalid proposer_index. let proposer_shuffling_root = signed_blob_sidecar.message.block_parent_root; @@ -229,6 +227,12 @@ pub fn validate_blob_sidecar_for_gossip( let (proposer_index, fork) = if let Some(proposer) = proposer_opt { (proposer.index, proposer.fork) } else { + debug!( + chain.log, + "Proposer shuffling cache miss for blob verification"; + "block_root" => %block_root, + "index" => %blob_index, + ); // The cached head state is in the same epoch as the blob or the state has already been // advanced to the blob's epoch let snapshot = &chain.canonical_head.cached_head().snapshot; @@ -242,6 +246,18 @@ pub fn validate_blob_sidecar_for_gossip( } // Need to advance the state to get the proposer index else { + // Reaching this condition too often might be an issue since we could theoretically have + // 5 threads (4 blob indices + 1 block) cloning the state. + // We shouldn't be seeing this condition a lot because we try to advance the state + // 3 seconds before the start of a slot. However, if this becomes an issue during testing, we should + // consider sending a blob for reprocessing to reduce the number of state clones. + warn!( + chain.log, + "Cached head not advanced for blob verification"; + "block_root" => %block_root, + "index" => %blob_index, + "action" => "contact the devs if you see this msg too often" + ); // The state produced is only valid for determining proposer/attester shuffling indices. let mut cloned_state = snapshot.clone_with(CloneConfig::committee_caches_only()); let state = cheap_state_advance_to_obtain_committees( diff --git a/beacon_node/beacon_chain/src/observed_blob_sidecars.rs b/beacon_node/beacon_chain/src/observed_blob_sidecars.rs index 1577d5951ca..3ff1789049e 100644 --- a/beacon_node/beacon_chain/src/observed_blob_sidecars.rs +++ b/beacon_node/beacon_chain/src/observed_blob_sidecars.rs @@ -97,3 +97,293 @@ impl ObservedBlobSidecars { self.items.retain(|k, _| k.1 > finalized_slot); } } + +#[cfg(test)] +mod tests { + use super::*; + use types::{BlobSidecar, Hash256, MainnetEthSpec}; + + type E = MainnetEthSpec; + + fn get_blob_sidecar(slot: u64, block_root: Hash256, index: u64) -> Arc> { + let mut blob_sidecar = BlobSidecar::empty(); + blob_sidecar.block_root = block_root; + blob_sidecar.slot = slot.into(); + blob_sidecar.index = index; + Arc::new(blob_sidecar) + } + + #[test] + fn pruning() { + let mut cache = ObservedBlobSidecars::default(); + + assert_eq!(cache.finalized_slot, 0, "finalized slot is zero"); + assert_eq!(cache.items.len(), 0, "no slots should be present"); + + // Slot 0, index 0 + let block_root_a = Hash256::random(); + let sidecar_a = get_blob_sidecar(0, block_root_a, 0); + + assert_eq!( + cache.observe_sidecar(&sidecar_a), + Ok(false), + "can observe proposer, indicates proposer unobserved" + ); + + /* + * Preconditions. + */ + + assert_eq!(cache.finalized_slot, 0, "finalized slot is zero"); + assert_eq!( + cache.items.len(), + 1, + "only one (slot, root) tuple should be present" + ); + assert_eq!( + cache + .items + .get(&(block_root_a, Slot::new(0))) + .expect("slot zero should be present") + .len(), + 1, + "only one item should be present" + ); + + /* + * Check that a prune at the genesis slot does nothing. + */ + + cache.prune(Slot::new(0)); + + assert_eq!(cache.finalized_slot, 0, "finalized slot is zero"); + assert_eq!(cache.items.len(), 1, "only one slot should be present"); + assert_eq!( + cache + .items + .get(&(block_root_a, Slot::new(0))) + .expect("slot zero should be present") + .len(), + 1, + "only one item should be present" + ); + + /* + * Check that a prune empties the cache + */ + + cache.prune(E::slots_per_epoch().into()); + assert_eq!( + cache.finalized_slot, + Slot::from(E::slots_per_epoch()), + "finalized slot is updated" + ); + assert_eq!(cache.items.len(), 0, "no items left"); + + /* + * Check that we can't insert a finalized sidecar + */ + + // First slot of finalized epoch + let block_b = get_blob_sidecar(E::slots_per_epoch(), Hash256::random(), 0); + + assert_eq!( + cache.observe_sidecar(&block_b), + Err(Error::FinalizedBlob { + slot: E::slots_per_epoch().into(), + finalized_slot: E::slots_per_epoch().into(), + }), + "cant insert finalized sidecar" + ); + + assert_eq!(cache.items.len(), 0, "sidecar was not added"); + + /* + * Check that we _can_ insert a non-finalized block + */ + + let three_epochs = E::slots_per_epoch() * 3; + + // First slot of finalized epoch + let block_root_b = Hash256::random(); + let block_b = get_blob_sidecar(three_epochs, block_root_b, 0); + + assert_eq!( + cache.observe_sidecar(&block_b), + Ok(false), + "can insert non-finalized block" + ); + + assert_eq!(cache.items.len(), 1, "only one slot should be present"); + assert_eq!( + cache + .items + .get(&(block_root_b, Slot::new(three_epochs))) + .expect("the three epochs slot should be present") + .len(), + 1, + "only one proposer should be present" + ); + + /* + * Check that a prune doesnt wipe later blocks + */ + + let two_epochs = E::slots_per_epoch() * 2; + cache.prune(two_epochs.into()); + + assert_eq!( + cache.finalized_slot, + Slot::from(two_epochs), + "finalized slot is updated" + ); + + assert_eq!(cache.items.len(), 1, "only one slot should be present"); + assert_eq!( + cache + .items + .get(&(block_root_b, Slot::new(three_epochs))) + .expect("the three epochs slot should be present") + .len(), + 1, + "only one proposer should be present" + ); + } + + #[test] + fn simple_observations() { + let mut cache = ObservedBlobSidecars::default(); + + // Slot 0, index 0 + let block_root_a = Hash256::random(); + let sidecar_a = get_blob_sidecar(0, block_root_a, 0); + + assert_eq!( + cache.is_known(&sidecar_a), + Ok(false), + "no observation in empty cache" + ); + + assert_eq!( + cache.observe_sidecar(&sidecar_a), + Ok(false), + "can observe proposer, indicates proposer unobserved" + ); + + assert_eq!( + cache.is_known(&sidecar_a), + Ok(true), + "observed block is indicated as true" + ); + + assert_eq!( + cache.observe_sidecar(&sidecar_a), + Ok(true), + "observing again indicates true" + ); + + assert_eq!(cache.finalized_slot, 0, "finalized slot is zero"); + assert_eq!(cache.items.len(), 1, "only one slot should be present"); + assert_eq!( + cache + .items + .get(&(block_root_a, Slot::new(0))) + .expect("slot zero should be present") + .len(), + 1, + "only one proposer should be present" + ); + + // Slot 1, proposer 0 + + let block_root_b = Hash256::random(); + let sidecar_b = get_blob_sidecar(1, block_root_b, 0); + + assert_eq!( + cache.is_known(&sidecar_b), + Ok(false), + "no observation for new slot" + ); + assert_eq!( + cache.observe_sidecar(&sidecar_b), + Ok(false), + "can observe proposer for new slot, indicates proposer unobserved" + ); + assert_eq!( + cache.is_known(&sidecar_b), + Ok(true), + "observed block in slot 1 is indicated as true" + ); + assert_eq!( + cache.observe_sidecar(&sidecar_b), + Ok(true), + "observing slot 1 again indicates true" + ); + + assert_eq!(cache.finalized_slot, 0, "finalized slot is zero"); + assert_eq!(cache.items.len(), 2, "two slots should be present"); + assert_eq!( + cache + .items + .get(&(block_root_a, Slot::new(0))) + .expect("slot zero should be present") + .len(), + 1, + "only one proposer should be present in slot 0" + ); + assert_eq!( + cache + .items + .get(&(block_root_b, Slot::new(1))) + .expect("slot zero should be present") + .len(), + 1, + "only one proposer should be present in slot 1" + ); + + // Slot 0, index 1 + let sidecar_c = get_blob_sidecar(0, block_root_a, 1); + + assert_eq!( + cache.is_known(&sidecar_c), + Ok(false), + "no observation for new index" + ); + assert_eq!( + cache.observe_sidecar(&sidecar_c), + Ok(false), + "can observe new index, indicates sidecar unobserved for new index" + ); + assert_eq!( + cache.is_known(&sidecar_c), + Ok(true), + "observed new sidecar is indicated as true" + ); + assert_eq!( + cache.observe_sidecar(&sidecar_c), + Ok(true), + "observing new sidecar again indicates true" + ); + + assert_eq!(cache.finalized_slot, 0, "finalized slot is zero"); + assert_eq!(cache.items.len(), 2, "two slots should be present"); + assert_eq!( + cache + .items + .get(&(block_root_a, Slot::new(0))) + .expect("slot zero should be present") + .len(), + 2, + "two blob indices should be present in slot 0" + ); + + // Try adding an out of bounds index + let invalid_index = E::max_blobs_per_block() as u64; + let sidecar_d = get_blob_sidecar(0, block_root_a, 4); + assert_eq!( + cache.observe_sidecar(&sidecar_d), + Err(Error::InvalidBlobIndex(invalid_index)), + "cannot add an index > MaxBlobsPerBlock" + ); + } +} From cbe488049077a4c3c535a3fb34ca9e104f223dab Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Wed, 26 Apr 2023 10:26:00 -0700 Subject: [PATCH 80/96] Fix deneb doppelganger tests (#4124) * Temp hack to compile * Fix doppelganger tests * Kill in groups instead of storing pid * Install geth in CI * Install geth first * Fix eth1_block_hash * Fix directory paths and block hash * Fix workflow for local testnets; reset genesis.json after running script * Disable capella and deneb forks for doppelganger tests * oops not capella * Spin up a spare bn for the doppelganger validator * testing * Revert "testing" This reverts commit 14eb178bca5b7d27b9cd9b665b5cd2c916f50901. * Modify beacon_node script to take trusted peers * Set doppelganger bn as a trusted peer * Update var * update another * Fix port * Add a flag to disable peer scoring * Disable peer scoring in local testnet bn script * Revert trusted peers hack * fmt * Fix proposer boost score --- .github/workflows/local-testnet.yml | 11 +- .github/workflows/test-suite.yml | 11 +- scripts/local_testnet/beacon_node.sh | 1 + scripts/local_testnet/el_bootnode.sh | 2 +- scripts/local_testnet/setup.sh | 6 +- scripts/local_testnet/vars.env | 2 +- scripts/tests/doppelganger_protection.sh | 51 +- scripts/tests/genesis.json | 855 +++++++++++++++++++++++ scripts/tests/vars.env | 28 +- 9 files changed, 921 insertions(+), 46 deletions(-) create mode 100644 scripts/tests/genesis.json diff --git a/.github/workflows/local-testnet.yml b/.github/workflows/local-testnet.yml index 66051af3b78..8b6728c795f 100644 --- a/.github/workflows/local-testnet.yml +++ b/.github/workflows/local-testnet.yml @@ -26,7 +26,16 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install ganache run: npm install ganache@latest --global - + - name: Install geth + run: | + sudo add-apt-repository -y ppa:ethereum/ethereum + sudo apt-get update + sudo apt-get install ethereum + if: matrix.os == 'ubuntu-22.04' + run: | + brew tap ethereum/ethereum + brew install ethereum + if: matrix.os == 'macos-12' - name: Install GNU sed & GNU grep run: | brew install gnu-sed grep diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index c970010ee60..aab5dafe4d4 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -260,8 +260,11 @@ jobs: uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install ganache - run: sudo npm install -g ganache + - name: Install geth + run: | + sudo add-apt-repository -y ppa:ethereum/ethereum + sudo apt-get update + sudo apt-get install ethereum - name: Install lighthouse and lcli run: | make @@ -269,11 +272,11 @@ jobs: - name: Run the doppelganger protection success test script run: | cd scripts/tests - ./doppelganger_protection.sh success + ./doppelganger_protection.sh success genesis.json - name: Run the doppelganger protection failure test script run: | cd scripts/tests - ./doppelganger_protection.sh failure + ./doppelganger_protection.sh failure genesis.json execution-engine-integration-ubuntu: name: execution-engine-integration-ubuntu runs-on: ubuntu-latest diff --git a/scripts/local_testnet/beacon_node.sh b/scripts/local_testnet/beacon_node.sh index 3738a05e8ae..1a04d12d4a0 100755 --- a/scripts/local_testnet/beacon_node.sh +++ b/scripts/local_testnet/beacon_node.sh @@ -53,6 +53,7 @@ exec $lighthouse_binary \ --datadir $data_dir \ --testnet-dir $TESTNET_DIR \ --enable-private-discovery \ + --disable-peer-scoring \ --staking \ --enr-address 127.0.0.1 \ --enr-udp-port $network_port \ diff --git a/scripts/local_testnet/el_bootnode.sh b/scripts/local_testnet/el_bootnode.sh index 1b8834b8904..ee0b43b82ab 100755 --- a/scripts/local_testnet/el_bootnode.sh +++ b/scripts/local_testnet/el_bootnode.sh @@ -2,4 +2,4 @@ priv_key="02fd74636e96a8ffac8e7b01b0de8dea94d6bcf4989513b38cf59eb32163ff91" source ./vars.env -$BOOTNODE_BINARY --nodekeyhex $priv_key \ No newline at end of file +$EL_BOOTNODE_BINARY --nodekeyhex $priv_key \ No newline at end of file diff --git a/scripts/local_testnet/setup.sh b/scripts/local_testnet/setup.sh index 5195d26750f..d405698f332 100755 --- a/scripts/local_testnet/setup.sh +++ b/scripts/local_testnet/setup.sh @@ -56,5 +56,7 @@ GENESIS_TIME=$(lcli pretty-ssz state_merge ~/.lighthouse/local-testnet/testnet/g CAPELLA_TIME=$((GENESIS_TIME + (CAPELLA_FORK_EPOCH * 32 * SECONDS_PER_SLOT))) DENEB_TIME=$((GENESIS_TIME + (DENEB_FORK_EPOCH * 32 * SECONDS_PER_SLOT))) -sed -i 's/"shanghaiTime".*$/"shanghaiTime": '"$CAPELLA_TIME"',/g' genesis.json -sed -i 's/"shardingForkTime".*$/"shardingForkTime": '"$DENEB_TIME"',/g' genesis.json +CURR_DIR=`pwd` + +sed -i 's/"shanghaiTime".*$/"shanghaiTime": '"$CAPELLA_TIME"',/g' $CURR_DIR/genesis.json +sed -i 's/"shardingForkTime".*$/"shardingForkTime": '"$DENEB_TIME"',/g' $CURR_DIR/genesis.json diff --git a/scripts/local_testnet/vars.env b/scripts/local_testnet/vars.env index 515153c785f..475bedea384 100644 --- a/scripts/local_testnet/vars.env +++ b/scripts/local_testnet/vars.env @@ -1,5 +1,5 @@ GETH_BINARY=geth -BOOTNODE_BINARY=bootnode +EL_BOOTNODE_BINARY=bootnode # Base directories for the validator keys and secrets DATADIR=~/.lighthouse/local-testnet diff --git a/scripts/tests/doppelganger_protection.sh b/scripts/tests/doppelganger_protection.sh index 95dfff56962..722d85a2842 100755 --- a/scripts/tests/doppelganger_protection.sh +++ b/scripts/tests/doppelganger_protection.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Requires `lighthouse`, ``lcli`, `ganache`, `curl`, `jq` +# Requires `lighthouse`, ``lcli`, `geth`, `curl`, `jq` BEHAVIOR=$1 @@ -15,21 +15,15 @@ exit_if_fails() { $@ EXIT_CODE=$? if [[ $EXIT_CODE -eq 1 ]]; then - exit 111 + exit 1 fi } +genesis_file=$2 source ./vars.env exit_if_fails ../local_testnet/clean.sh -echo "Starting ganache" - -exit_if_fails ../local_testnet/ganache_test_node.sh &> /dev/null & -GANACHE_PID=$! - -# Wait for ganache to start -sleep 5 echo "Setting up local testnet" @@ -41,28 +35,33 @@ exit_if_fails cp -R $HOME/.lighthouse/local-testnet/node_1 $HOME/.lighthouse/loc echo "Starting bootnode" exit_if_fails ../local_testnet/bootnode.sh &> /dev/null & -BOOT_PID=$! + +exit_if_fails ../local_testnet/el_bootnode.sh &> /dev/null & # wait for the bootnode to start sleep 10 -echo "Starting local beacon nodes" +echo "Starting local execution nodes" -exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_1 9000 8000 &> /dev/null & -BEACON_PID=$! -exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_2 9100 8100 &> /dev/null & -BEACON_PID2=$! -exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_3 9200 8200 &> /dev/null & -BEACON_PID3=$! +exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir1 7000 6000 5000 $genesis_file &> geth.log & +exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir2 7100 6100 5100 $genesis_file &> /dev/null & +exit_if_fails ../local_testnet/geth.sh $HOME/.lighthouse/local-testnet/geth_datadir3 7200 6200 5200 $genesis_file &> /dev/null & + +sleep 20 + +# Reset the `genesis.json` config file fork times. +sed -i 's/"shanghaiTime".*$/"shanghaiTime": 0,/g' genesis.json +sed -i 's/"shardingForkTime".*$/"shardingForkTime": 0,/g' genesis.json + +exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_1 9000 8000 http://localhost:5000 $HOME/.lighthouse/local-testnet/geth_datadir1/geth/jwtsecret &> /dev/null & +exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_2 9100 8100 http://localhost:5100 $HOME/.lighthouse/local-testnet/geth_datadir2/geth/jwtsecret &> beacon1.log & +exit_if_fails ../local_testnet/beacon_node.sh $HOME/.lighthouse/local-testnet/node_3 9200 8200 http://localhost:5200 $HOME/.lighthouse/local-testnet/geth_datadir3/geth/jwtsecret &> /dev/null & echo "Starting local validator clients" exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1 http://localhost:8000 &> /dev/null & -VALIDATOR_1_PID=$! exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_2 http://localhost:8100 &> /dev/null & -VALIDATOR_2_PID=$! exit_if_fails ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_3 http://localhost:8200 &> /dev/null & -VALIDATOR_3_PID=$! echo "Waiting an epoch before starting the next validator client" sleep $(( $SECONDS_PER_SLOT * 32 )) @@ -71,7 +70,7 @@ if [[ "$BEHAVIOR" == "failure" ]]; then echo "Starting the doppelganger validator client" - # Use same keys as keys from VC1, but connect to BN2 + # Use same keys as keys from VC1 and connect to BN2 # This process should not last longer than 2 epochs timeout $(( $SECONDS_PER_SLOT * 32 * 2 )) ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_1_doppelganger http://localhost:8100 DOPPELGANGER_EXIT=$? @@ -79,7 +78,9 @@ if [[ "$BEHAVIOR" == "failure" ]]; then echo "Shutting down" # Cleanup - kill $BOOT_PID $BEACON_PID $BEACON_PID2 $BEACON_PID3 $GANACHE_PID $VALIDATOR_1_PID $VALIDATOR_2_PID $VALIDATOR_3_PID + killall geth + killall lighthouse + killall bootnode echo "Done" @@ -98,7 +99,6 @@ if [[ "$BEHAVIOR" == "success" ]]; then echo "Starting the last validator client" ../local_testnet/validator_client.sh $HOME/.lighthouse/local-testnet/node_4 http://localhost:8100 & - VALIDATOR_4_PID=$! DOPPELGANGER_FAILURE=0 # Sleep three epochs, then make sure all validators were active in epoch 2. Use @@ -144,7 +144,10 @@ if [[ "$BEHAVIOR" == "success" ]]; then # Cleanup cd $PREVIOUS_DIR - kill $BOOT_PID $BEACON_PID $BEACON_PID2 $BEACON_PID3 $GANACHE_PID $VALIDATOR_1_PID $VALIDATOR_2_PID $VALIDATOR_3_PID $VALIDATOR_4_PID + + killall geth + killall lighthouse + killall bootnode echo "Done" diff --git a/scripts/tests/genesis.json b/scripts/tests/genesis.json new file mode 100644 index 00000000000..751176048cd --- /dev/null +++ b/scripts/tests/genesis.json @@ -0,0 +1,855 @@ +{ + "config": { + "chainId": 4242, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "mergeNetsplitBlock": 0, + "shanghaiTime": 0, + "shardingForkTime": 0, + "terminalTotalDifficulty": 0 + }, + "alloc": { + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x6d6172697573766477000000" + }, + "0x0000000000000000000000000000000000000000": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000001": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000002": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000003": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000004": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000005": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000006": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000007": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000008": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000009": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000010": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000011": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000012": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000013": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000014": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000015": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000016": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000017": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000018": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000019": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000020": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000021": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000022": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000023": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000024": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000025": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000026": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000027": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000028": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000029": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000030": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000031": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000032": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000033": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000034": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000035": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000036": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000037": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000038": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000039": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000040": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000041": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000042": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000043": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000044": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000045": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000046": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000047": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000048": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000049": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000050": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000051": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000052": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000053": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000054": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000055": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000056": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000057": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000058": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000059": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000060": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000061": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000062": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000063": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000064": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000065": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000066": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000067": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000068": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000069": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000070": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000071": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000072": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000073": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000074": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000075": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000076": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000077": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000078": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000079": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000080": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000081": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000082": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000083": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000084": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000085": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000086": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000087": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000088": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000089": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000090": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000091": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000092": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000093": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000094": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000095": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000096": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000097": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000098": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000099": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009f": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000aa": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ab": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ac": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ad": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ae": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000af": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ba": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000be": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bf": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ca": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ce": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cf": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000da": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000db": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000dc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000dd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000de": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000df": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ea": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000eb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ec": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ed": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ee": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ef": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fa": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fe": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ff": { + "balance": "1" + }, + "0x4242424242424242424242424242424242424242": { + "balance": "0", + "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a26469706673582212201dd26f37a621703009abf16e77e69c93dc50c79db7f6cc37543e3e0e3decdc9764736f6c634300060b0033", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" + } + }, + "0x9a4aa7d9C2F6386e5F24d790eB2FFB9fd543A170": { + "balance": "1000000000000000000000000000" + }, + "0x5E3141B900ac5f5608b0d057D10d45a0e4927cD9": { + "balance": "1000000000000000000000000000" + }, + "0x7cF5Dbc49F0904065664b5B6C0d69CaB55F33988": { + "balance": "1000000000000000000000000000" + }, + "0x8D12b071A6F3823A535D38C4a583a2FA1859e822": { + "balance": "1000000000000000000000000000" + }, + "0x3B575D3cda6b30736A38B031E0d245E646A21135": { + "balance": "1000000000000000000000000000" + }, + "0x53bDe6CF93461674F590E532006b4022dA57A724": { + "balance": "1000000000000000000000000000" + } + }, + "coinbase": "0x0000000000000000000000000000000000000000", + "difficulty": "0x01", + "extraData": "", + "gasLimit": "0x400000", + "nonce": "0x1234", + "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "1662465600" +} diff --git a/scripts/tests/vars.env b/scripts/tests/vars.env index b656492b3cf..6c39aeb9971 100644 --- a/scripts/tests/vars.env +++ b/scripts/tests/vars.env @@ -1,17 +1,23 @@ +# Path to the geth binary +GETH_BINARY=geth +EL_BOOTNODE_BINARY=bootnode + # Base directories for the validator keys and secrets DATADIR=~/.lighthouse/local-testnet # Directory for the eth2 config TESTNET_DIR=$DATADIR/testnet -# Mnemonic for the ganache test network -ETH1_NETWORK_MNEMONIC="vast thought differ pull jewel broom cook wrist tribe word before omit" +EL_BOOTNODE_ENODE="enode://51ea9bb34d31efc3491a842ed13b8cab70e753af108526b57916d716978b380ed713f4336a80cdb85ec2a115d5a8c0ae9f3247bed3c84d3cb025c6bab311062c@127.0.0.1:0?discport=30301" -# Hardcoded deposit contract based on ETH1_NETWORK_MNEMONIC -DEPOSIT_CONTRACT_ADDRESS=8c594691c0e592ffa21f153a16ae41db5befcaaa +# Hardcoded deposit contract +DEPOSIT_CONTRACT_ADDRESS=4242424242424242424242424242424242424242 GENESIS_FORK_VERSION=0x42424242 +# Block hash generated from genesis.json in directory +ETH1_BLOCK_HASH=4c2221e15760fd06c8c7a5202258c67e3d9e4aedf6db3a886ce9dc36938ad8d0 + VALIDATOR_COUNT=80 GENESIS_VALIDATOR_COUNT=80 @@ -33,11 +39,13 @@ BOOTNODE_PORT=4242 CHAIN_ID=4242 # Hard fork configuration -ALTAIR_FORK_EPOCH=18446744073709551615 -BELLATRIX_FORK_EPOCH=18446744073709551615 -CAPELLA_FORK_EPOCH=18446744073709551615 +ALTAIR_FORK_EPOCH=0 +BELLATRIX_FORK_EPOCH=0 +CAPELLA_FORK_EPOCH=1 DENEB_FORK_EPOCH=18446744073709551615 +TTD=0 + # Spec version (mainnet or minimal) SPEC_PRESET=mainnet @@ -52,9 +60,3 @@ PROPOSER_SCORE_BOOST=40 # Enable doppelganger detection VC_ARGS=" --enable-doppelganger-protection " - -# Using value of DEFAULT_TERMINAL_DIFFICULTY. -TTD=6400 - -# Using value of DEFAULT_ETH1_BLOCK_HASH. -ETH1_BLOCK_HASH="0x4242424242424242424242424242424242424242424242424242424242424242" From aa34339298e746fd803c418222a5d11547c6cc03 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Wed, 26 Apr 2023 13:53:06 -0500 Subject: [PATCH 81/96] Rename to MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS (#4206) Co-authored-by: realbigsean --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 ++-- beacon_node/beacon_chain/src/data_availability_checker.rs | 4 ++-- beacon_node/client/src/config.rs | 2 +- beacon_node/store/src/hot_cold_store.rs | 4 ++-- consensus/types/src/consts.rs | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 9f3585eb4a8..1ea00762041 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -117,7 +117,7 @@ use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; use types::blob_sidecar::{BlobIdentifier, BlobSidecarList, Blobs}; -use types::consts::deneb::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; +use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; use types::*; pub type ForkChoiceError = fork_choice::Error; @@ -6231,7 +6231,7 @@ impl BeaconChain { self.epoch().ok().map(|current_epoch| { std::cmp::max( fork_epoch, - current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS), + current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS), ) }) }) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index d9446e1ec0f..1b44947c08a 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::{BlobIdentifier, BlobSidecar}; -use types::consts::deneb::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; +use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; use types::{ BeaconBlockRef, BlobSidecarList, ChainSpec, Epoch, EthSpec, ExecPayload, FullPayload, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, @@ -441,7 +441,7 @@ impl DataAvailabilityChecker { .map(|current_epoch| { std::cmp::max( fork_epoch, - current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS), + current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS), ) }) }) diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 8ce0fa2cc5f..0be6bddfc03 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -52,7 +52,7 @@ pub struct Config { /// Path where the blobs database will be located if blobs should be in a separate database. /// /// The capacity this location should hold varies with the data availability boundary. It - /// should be able to store < 69 GB when [MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS](types::consts::deneb::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS) is 4096 + /// should be able to store < 69 GB when [MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS](types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS) is 4096 /// epochs of 32 slots (up to 131072 bytes data per blob and up to 4 blobs per block, 88 bytes /// of [BlobsSidecar](types::BlobsSidecar) metadata per block). pub blobs_db_path: Option, diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 96134537797..8071f3eea2d 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -39,7 +39,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use types::blob_sidecar::BlobSidecarList; -use types::consts::deneb::MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; +use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; use types::*; /// On-disk database that stores finalized states efficiently. @@ -1866,7 +1866,7 @@ impl, Cold: ItemStore> HotColdDB let min_current_epoch = self.get_split_slot().epoch(E::slots_per_epoch()) + Epoch::new(2); let min_data_availability_boundary = std::cmp::max( deneb_fork, - min_current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS), + min_current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS), ); self.try_prune_blobs(force, Some(min_data_availability_boundary)) diff --git a/consensus/types/src/consts.rs b/consensus/types/src/consts.rs index 6c60a985a5e..34398d88771 100644 --- a/consensus/types/src/consts.rs +++ b/consensus/types/src/consts.rs @@ -32,7 +32,7 @@ pub mod deneb { "52435875175126190479447740508185965837690552500527637822603658699938581184513" ) .expect("should initialize BLS_MODULUS"); - pub static ref MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS: Epoch = Epoch::from(4096_u64); + pub static ref MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: Epoch = Epoch::from(4096_u64); } pub const BLOB_TX_TYPE: u8 = 3; pub const VERSIONED_HASH_VERSION_KZG: u8 = 1; From c1d47da02de79157156677a49511a503ff0e333f Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:18:21 -0500 Subject: [PATCH 82/96] Update `engine_api` to latest version (#4223) * Update Engine API to Latest * Get Mock EE Working * Fix Mock EE * Update Engine API Again * Rip out get_blobs_bundle Stuff * Fix Test Harness * Fix Clippy Complaints * Fix Beacon Chain Tests --- Cargo.lock | 1 + beacon_node/beacon_chain/src/beacon_chain.rs | 20 ++- .../beacon_chain/src/blob_verification.rs | 20 ++- beacon_node/beacon_chain/src/test_utils.rs | 82 ++++++++++-- .../beacon_chain/tests/block_verification.rs | 10 +- .../tests/payload_invalidation.rs | 16 ++- beacon_node/beacon_chain/tests/store_tests.rs | 8 +- beacon_node/execution_layer/Cargo.toml | 1 + beacon_node/execution_layer/src/engine_api.rs | 20 ++- .../execution_layer/src/engine_api/http.rs | 20 --- .../src/engine_api/json_structures.rs | 32 ++++- beacon_node/execution_layer/src/lib.rs | 124 +++++++++--------- .../test_utils/execution_block_generator.rs | 124 +++++++++++++++++- .../src/test_utils/handle_rpc.rs | 10 +- .../src/test_utils/mock_execution_layer.rs | 4 + .../execution_layer/src/test_utils/mod.rs | 12 +- .../http_api/tests/interactive_tests.rs | 20 ++- .../network/src/beacon_processor/tests.rs | 16 +-- consensus/fork_choice/tests/tests.rs | 22 ++-- consensus/types/src/blob_sidecar.rs | 29 +++- consensus/types/src/lib.rs | 1 + crypto/kzg/src/kzg_commitment.rs | 13 +- crypto/kzg/src/lib.rs | 1 + testing/node_test_rig/src/lib.rs | 2 +- 24 files changed, 449 insertions(+), 159 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fad095449f3..89780448df3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2647,6 +2647,7 @@ dependencies = [ "hex", "jsonwebtoken", "keccak-hash", + "kzg", "lazy_static", "lighthouse_metrics", "lru 0.7.8", diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 1ea00762041..e7180cae141 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4711,7 +4711,7 @@ impl BeaconChain { bls_to_execution_changes, } = partial_beacon_block; - let (inner_block, blobs_opt) = match &state { + let (inner_block, blobs_opt, proofs_opt) = match &state { BeaconState::Base(_) => ( BeaconBlock::Base(BeaconBlockBase { slot, @@ -4731,6 +4731,7 @@ impl BeaconChain { }, }), None, + None, ), BeaconState::Altair(_) => ( BeaconBlock::Altair(BeaconBlockAltair { @@ -4753,9 +4754,10 @@ impl BeaconChain { }, }), None, + None, ), BeaconState::Merge(_) => { - let (payload, _, _) = block_contents + let (payload, _, _, _) = block_contents .ok_or(BlockProductionError::MissingExecutionPayload)? .deconstruct(); ( @@ -4781,10 +4783,11 @@ impl BeaconChain { }, }), None, + None, ) } BeaconState::Capella(_) => { - let (payload, _, _) = block_contents + let (payload, _, _, _) = block_contents .ok_or(BlockProductionError::MissingExecutionPayload)? .deconstruct(); ( @@ -4811,10 +4814,11 @@ impl BeaconChain { }, }), None, + None, ) } BeaconState::Deneb(_) => { - let (payload, kzg_commitments, blobs) = block_contents + let (payload, kzg_commitments, blobs, proofs) = block_contents .ok_or(BlockProductionError::MissingExecutionPayload)? .deconstruct(); ( @@ -4843,6 +4847,7 @@ impl BeaconChain { }, }), blobs, + proofs, ) } }; @@ -4915,8 +4920,11 @@ impl BeaconChain { ))); } - let kzg_proofs = - Self::compute_blob_kzg_proofs(kzg, &blobs, expected_kzg_commitments, slot)?; + let kzg_proofs = if let Some(proofs) = proofs_opt { + Vec::from(proofs) + } else { + Self::compute_blob_kzg_proofs(kzg, &blobs, expected_kzg_commitments, slot)? + }; kzg_utils::validate_blobs::( kzg, diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 1e561cf34b8..d5e5e9665a5 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -12,13 +12,14 @@ use crate::data_availability_checker::{ }; use crate::kzg_utils::{validate_blob, validate_blobs}; use crate::BeaconChainError; +use eth2::types::BlockContentsTuple; use kzg::Kzg; use slog::{debug, warn}; use std::borrow::Cow; use types::{ BeaconBlockRef, BeaconState, BeaconStateError, BlobSidecar, BlobSidecarList, ChainSpec, - CloneConfig, Epoch, EthSpec, Hash256, KzgCommitment, RelativeEpoch, SignedBeaconBlock, - SignedBeaconBlockHeader, SignedBlobSidecar, Slot, + CloneConfig, Epoch, EthSpec, FullPayload, Hash256, KzgCommitment, RelativeEpoch, + SignedBeaconBlock, SignedBeaconBlockHeader, SignedBlobSidecar, Slot, }; #[derive(Debug)] @@ -659,3 +660,18 @@ impl From> for BlockWrapper { Self::Block(Arc::new(value)) } } + +impl From>> for BlockWrapper { + fn from(value: BlockContentsTuple>) -> Self { + match value.1 { + Some(variable_list) => Self::BlockAndBlobs( + Arc::new(value.0), + Vec::from(variable_list) + .into_iter() + .map(|signed_blob| signed_blob.message) + .collect::>(), + ), + None => Self::Block(Arc::new(value.0)), + } + } +} diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index c54c3df656b..db502fcda5a 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1,3 +1,4 @@ +use crate::blob_verification::{AsBlock, BlockWrapper}; pub use crate::persisted_beacon_chain::PersistedBeaconChain; pub use crate::{ beacon_chain::{BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, FORK_CHOICE_DB_KEY, OP_POOL_DB_KEY}, @@ -13,6 +14,7 @@ use crate::{ StateSkipConfig, }; use bls::get_withdrawal_credentials; +use eth2::types::BlockContentsTuple; use execution_layer::{ auth::JwtKey, test_utils::{ @@ -25,7 +27,7 @@ use fork_choice::CountUnrealized; use futures::channel::mpsc::Receiver; pub use genesis::{interop_genesis_state_with_eth1, DEFAULT_ETH1_BLOCK_HASH}; use int_to_bytes::int_to_bytes32; -use kzg::TrustedSetup; +use kzg::{Kzg, TrustedSetup}; use merkle_proof::MerkleTree; use parking_lot::Mutex; use parking_lot::RwLockWriteGuard; @@ -446,6 +448,13 @@ where let deneb_time = spec.deneb_fork_epoch.map(|epoch| { HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); + + let trusted_setup: TrustedSetup = + serde_json::from_reader(eth2_network_config::TRUSTED_SETUP) + .map_err(|e| format!("Unable to read trusted setup file: {}", e)) + .expect("should have trusted setup"); + let kzg = Kzg::new_from_trusted_setup(trusted_setup).expect("should create kzg"); + let mock = MockExecutionLayer::new( self.runtime.task_executor.clone(), DEFAULT_TERMINAL_BLOCK, @@ -455,6 +464,7 @@ where Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec, None, + Some(kzg), ); self.execution_layer = Some(mock.el.clone()); self.mock_execution_layer = Some(mock); @@ -477,6 +487,11 @@ where let deneb_time = spec.deneb_fork_epoch.map(|epoch| { HARNESS_GENESIS_TIME + spec.seconds_per_slot * E::slots_per_epoch() * epoch.as_u64() }); + let trusted_setup: TrustedSetup = + serde_json::from_reader(eth2_network_config::TRUSTED_SETUP) + .map_err(|e| format!("Unable to read trusted setup file: {}", e)) + .expect("should have trusted setup"); + let kzg = Kzg::new_from_trusted_setup(trusted_setup).expect("should create kzg"); let mock_el = MockExecutionLayer::new( self.runtime.task_executor.clone(), DEFAULT_TERMINAL_BLOCK, @@ -486,6 +501,7 @@ where Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec.clone(), Some(builder_url.clone()), + Some(kzg), ) .move_to_terminal_block(); @@ -755,7 +771,7 @@ where &self, mut state: BeaconState, slot: Slot, - ) -> (SignedBeaconBlock, BeaconState) { + ) -> (BlockContentsTuple>, BeaconState) { assert_ne!(slot, 0, "can't produce a block at slot 0"); assert!(slot >= state.slot()); @@ -795,7 +811,37 @@ where &self.spec, ); - (signed_block, state) + let block_contents: BlockContentsTuple> = match &signed_block { + SignedBeaconBlock::Base(_) + | SignedBeaconBlock::Altair(_) + | SignedBeaconBlock::Merge(_) + | SignedBeaconBlock::Capella(_) => (signed_block, None), + SignedBeaconBlock::Deneb(_) => { + if let Some(blobs) = self + .chain + .proposal_blob_cache + .pop(&signed_block.canonical_root()) + { + let signed_blobs = Vec::from(blobs) + .into_iter() + .map(|blob| { + blob.sign( + &self.validator_keypairs[proposer_index].sk, + &state.fork(), + state.genesis_validators_root(), + &self.spec, + ) + }) + .collect::>() + .into(); + (signed_block, Some(signed_blobs)) + } else { + (signed_block, None) + } + } + }; + + (block_contents, state) } /// Useful for the `per_block_processing` tests. Creates a block, and returns the state after @@ -1663,18 +1709,18 @@ where (deposits, state) } - pub async fn process_block( + pub async fn process_block>>( &self, slot: Slot, block_root: Hash256, - block: SignedBeaconBlock, + block: B, ) -> Result> { self.set_current_slot(slot); let block_hash: SignedBeaconBlockHash = self .chain .process_block( block_root, - Arc::new(block), + block.into(), CountUnrealized::True, NotifyExecutionLayer::Yes, ) @@ -1685,15 +1731,16 @@ where Ok(block_hash) } - pub async fn process_block_result( + pub async fn process_block_result>>( &self, - block: SignedBeaconBlock, + block: B, ) -> Result> { + let wrapped_block = block.into(); let block_hash: SignedBeaconBlockHash = self .chain .process_block( - block.canonical_root(), - Arc::new(block), + wrapped_block.canonical_root(), + wrapped_block, CountUnrealized::True, NotifyExecutionLayer::Yes, ) @@ -1759,11 +1806,18 @@ where &self, slot: Slot, state: BeaconState, - ) -> Result<(SignedBeaconBlockHash, SignedBeaconBlock, BeaconState), BlockError> { + ) -> Result< + ( + SignedBeaconBlockHash, + BlockContentsTuple>, + BeaconState, + ), + BlockError, + > { self.set_current_slot(slot); let (block, new_state) = self.make_block(state, slot).await; let block_hash = self - .process_block(slot, block.canonical_root(), block.clone()) + .process_block(slot, block.0.canonical_root(), block.clone()) .await?; Ok((block_hash, block, new_state)) } @@ -1819,7 +1873,7 @@ where sync_committee_strategy: SyncCommitteeStrategy, ) -> Result<(SignedBeaconBlockHash, BeaconState), BlockError> { let (block_hash, block, state) = self.add_block_at_slot(slot, state).await?; - self.attest_block(&state, state_root, block_hash, &block, validators); + self.attest_block(&state, state_root, block_hash, &block.0, validators); if sync_committee_strategy == SyncCommitteeStrategy::AllValidators && state.current_sync_committee().is_ok() @@ -2047,7 +2101,7 @@ where state: BeaconState, slot: Slot, _block_strategy: BlockStrategy, - ) -> (SignedBeaconBlock, BeaconState) { + ) -> (BlockContentsTuple>, BeaconState) { self.make_block(state, slot).await } diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 88364d6ff5f..4b6d5b24120 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -1025,8 +1025,8 @@ async fn verify_block_for_gossip_slashing_detection() { harness.advance_slot(); let state = harness.get_current_state(); - let (block1, _) = harness.make_block(state.clone(), Slot::new(1)).await; - let (block2, _) = harness.make_block(state, Slot::new(1)).await; + let ((block1, _), _) = harness.make_block(state.clone(), Slot::new(1)).await; + let ((block2, _), _) = harness.make_block(state, Slot::new(1)).await; let verified_block = harness .chain @@ -1065,7 +1065,7 @@ async fn verify_block_for_gossip_doppelganger_detection() { let harness = get_harness(VALIDATOR_COUNT); let state = harness.get_current_state(); - let (block, _) = harness.make_block(state.clone(), Slot::new(1)).await; + let ((block, _), _) = harness.make_block(state.clone(), Slot::new(1)).await; let verified_block = harness .chain @@ -1152,7 +1152,7 @@ async fn add_base_block_to_altair_chain() { // Produce an Altair block. let state = harness.get_current_state(); let slot = harness.get_current_slot(); - let (altair_signed_block, _) = harness.make_block(state.clone(), slot).await; + let ((altair_signed_block, _), _) = harness.make_block(state.clone(), slot).await; let altair_block = &altair_signed_block .as_altair() .expect("test expects an altair block") @@ -1289,7 +1289,7 @@ async fn add_altair_block_to_base_chain() { // Produce an altair block. let state = harness.get_current_state(); let slot = harness.get_current_slot(); - let (base_signed_block, _) = harness.make_block(state.clone(), slot).await; + let ((base_signed_block, _), _) = harness.make_block(state.clone(), slot).await; let base_block = &base_signed_block .as_base() .expect("test expects a base block") diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index dbb2ebfaea5..e85b021f593 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -223,7 +223,7 @@ impl InvalidPayloadRig { let head = self.harness.chain.head_snapshot(); let state = head.beacon_state.clone_with_only_committee_caches(); let slot = slot_override.unwrap_or(state.slot() + 1); - let (block, post_state) = self.harness.make_block(state, slot).await; + let ((block, _), post_state) = self.harness.make_block(state, slot).await; let block_root = block.canonical_root(); let set_new_payload = |payload: Payload| match payload { @@ -691,7 +691,8 @@ async fn invalidates_all_descendants() { .state_at_slot(fork_parent_slot, StateSkipConfig::WithStateRoots) .unwrap(); assert_eq!(fork_parent_state.slot(), fork_parent_slot); - let (fork_block, _fork_post_state) = rig.harness.make_block(fork_parent_state, fork_slot).await; + let ((fork_block, _), _fork_post_state) = + rig.harness.make_block(fork_parent_state, fork_slot).await; let fork_block_root = rig .harness .chain @@ -789,7 +790,8 @@ async fn switches_heads() { .state_at_slot(fork_parent_slot, StateSkipConfig::WithStateRoots) .unwrap(); assert_eq!(fork_parent_state.slot(), fork_parent_slot); - let (fork_block, _fork_post_state) = rig.harness.make_block(fork_parent_state, fork_slot).await; + let ((fork_block, _), _fork_post_state) = + rig.harness.make_block(fork_parent_state, fork_slot).await; let fork_parent_root = fork_block.parent_root(); let fork_block_root = rig .harness @@ -1033,8 +1035,8 @@ async fn invalid_parent() { // Produce another block atop the parent, but don't import yet. let slot = parent_block.slot() + 1; rig.harness.set_current_slot(slot); - let (block, state) = rig.harness.make_block(parent_state, slot).await; - let block = Arc::new(block); + let (block_tuple, state) = rig.harness.make_block(parent_state, slot).await; + let block = Arc::new(block_tuple.0); let block_root = block.canonical_root(); assert_eq!(block.parent_root(), parent_root); @@ -1850,8 +1852,8 @@ impl InvalidHeadSetup { .chain .state_at_slot(slot - 1, StateSkipConfig::WithStateRoots) .unwrap(); - let (fork_block, _) = rig.harness.make_block(parent_state, slot).await; - opt_fork_block = Some(Arc::new(fork_block)); + let (fork_block_tuple, _) = rig.harness.make_block(parent_state, slot).await; + opt_fork_block = Some(Arc::new(fork_block_tuple.0)); } else { // Skipped slot. }; diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 33cb58fa790..239ae180699 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -2022,7 +2022,7 @@ async fn garbage_collect_temp_states_from_failed_block() { let genesis_state = harness.get_current_state(); let block_slot = Slot::new(2 * slots_per_epoch); - let (signed_block, state) = harness.make_block(genesis_state, block_slot).await; + let ((signed_block, _), state) = harness.make_block(genesis_state, block_slot).await; let (mut block, _) = signed_block.deconstruct(); @@ -2422,7 +2422,7 @@ async fn revert_minority_fork_on_resume() { harness1.process_attestations(attestations.clone()); harness2.process_attestations(attestations); - let (block, new_state) = harness1.make_block(state, slot).await; + let ((block, _), new_state) = harness1.make_block(state, slot).await; harness1 .process_block(slot, block.canonical_root(), block.clone()) @@ -2463,7 +2463,7 @@ async fn revert_minority_fork_on_resume() { harness2.process_attestations(attestations); // Minority chain block (no attesters). - let (block1, new_state1) = harness1.make_block(state1, slot).await; + let ((block1, _), new_state1) = harness1.make_block(state1, slot).await; harness1 .process_block(slot, block1.canonical_root(), block1) .await @@ -2471,7 +2471,7 @@ async fn revert_minority_fork_on_resume() { state1 = new_state1; // Majority chain block (all attesters). - let (block2, new_state2) = harness2.make_block(state2, slot).await; + let ((block2, _), new_state2) = harness2.make_block(state2, slot).await; harness2 .process_block(slot, block2.canonical_root(), block2.clone()) .await diff --git a/beacon_node/execution_layer/Cargo.toml b/beacon_node/execution_layer/Cargo.toml index 1b687a8b60e..d001a482da4 100644 --- a/beacon_node/execution_layer/Cargo.toml +++ b/beacon_node/execution_layer/Cargo.toml @@ -25,6 +25,7 @@ hex = "0.4.2" eth2_ssz = "0.4.1" eth2_ssz_types = "0.2.2" eth2 = { path = "../../common/eth2" } +kzg = { path = "../../crypto/kzg" } state_processing = { path = "../../consensus/state_processing" } superstruct = "0.6.0" lru = "0.7.1" diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 02d9e814988..cb09d3a0b97 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -16,12 +16,14 @@ use serde::{Deserialize, Serialize}; use std::convert::TryFrom; use strum::IntoStaticStr; use superstruct::superstruct; +use types::beacon_block_body::KzgCommitments; +use types::blob_sidecar::Blobs; pub use types::{ Address, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, Withdrawal, Withdrawals, }; -use types::{ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge}; +use types::{ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, KzgProofs}; pub mod auth; pub mod http; @@ -377,6 +379,8 @@ pub struct GetPayloadResponse { #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] pub execution_payload: ExecutionPayloadDeneb, pub block_value: Uint256, + #[superstruct(only(Deneb))] + pub blobs_bundle: BlobsBundleV1, } impl<'a, T: EthSpec> From> for ExecutionPayloadRef<'a, T> { @@ -395,20 +399,25 @@ impl From> for ExecutionPayload { } } -impl From> for (ExecutionPayload, Uint256) { +impl From> + for (ExecutionPayload, Uint256, Option>) +{ fn from(response: GetPayloadResponse) -> Self { match response { GetPayloadResponse::Merge(inner) => ( ExecutionPayload::Merge(inner.execution_payload), inner.block_value, + None, ), GetPayloadResponse::Capella(inner) => ( ExecutionPayload::Capella(inner.execution_payload), inner.block_value, + None, ), GetPayloadResponse::Deneb(inner) => ( ExecutionPayload::Deneb(inner.execution_payload), inner.block_value, + Some(inner.blobs_bundle), ), } } @@ -513,6 +522,13 @@ impl ExecutionPayloadBodyV1 { } } +#[derive(Clone, Default, Debug, PartialEq)] +pub struct BlobsBundleV1 { + pub commitments: KzgCommitments, + pub proofs: KzgProofs, + pub blobs: Blobs, +} + #[derive(Clone, Copy, Debug)] pub struct EngineCapabilities { pub new_payload_v1: bool, diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index b4777940738..137ba5318dd 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -40,9 +40,6 @@ pub const ENGINE_GET_PAYLOAD_V2: &str = "engine_getPayloadV2"; pub const ENGINE_GET_PAYLOAD_V3: &str = "engine_getPayloadV3"; pub const ENGINE_GET_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(2); -pub const ENGINE_GET_BLOBS_BUNDLE_V1: &str = "engine_getBlobsBundleV1"; -pub const ENGINE_GET_BLOBS_BUNDLE_TIMEOUT: Duration = Duration::from_secs(2); - pub const ENGINE_FORKCHOICE_UPDATED_V1: &str = "engine_forkchoiceUpdatedV1"; pub const ENGINE_FORKCHOICE_UPDATED_V2: &str = "engine_forkchoiceUpdatedV2"; pub const ENGINE_FORKCHOICE_UPDATED_TIMEOUT: Duration = Duration::from_secs(8); @@ -927,23 +924,6 @@ impl HttpJsonRpc { } } - pub async fn get_blobs_bundle_v1( - &self, - payload_id: PayloadId, - ) -> Result, Error> { - let params = json!([JsonPayloadIdRequest::from(payload_id)]); - - let response: JsonBlobsBundle = self - .rpc_request( - ENGINE_GET_BLOBS_BUNDLE_V1, - params, - ENGINE_GET_BLOBS_BUNDLE_TIMEOUT, - ) - .await?; - - Ok(response) - } - pub async fn forkchoice_updated_v1( &self, forkchoice_state: ForkchoiceState, diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index d7d9aae2987..6f35b528510 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -291,6 +291,8 @@ pub struct JsonGetPayloadResponse { pub execution_payload: JsonExecutionPayloadV3, #[serde(with = "eth2_serde_utils::u256_hex_be")] pub block_value: Uint256, + #[superstruct(only(V3))] + pub blobs_bundle: JsonBlobsBundleV1, } impl From> for GetPayloadResponse { @@ -312,6 +314,7 @@ impl From> for GetPayloadResponse { GetPayloadResponse::Deneb(GetPayloadResponseDeneb { execution_payload: response.execution_payload.into(), block_value: response.block_value, + blobs_bundle: response.blobs_bundle.into(), }) } } @@ -409,12 +412,31 @@ impl From for PayloadAttributes { } #[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(bound = "T: EthSpec", rename_all = "camelCase")] -pub struct JsonBlobsBundle { - pub block_hash: ExecutionBlockHash, - pub kzgs: KzgCommitments, +#[serde(bound = "E: EthSpec", rename_all = "camelCase")] +pub struct JsonBlobsBundleV1 { + pub commitments: KzgCommitments, + pub proofs: KzgProofs, #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] - pub blobs: Blobs, + pub blobs: Blobs, +} + +impl From> for JsonBlobsBundleV1 { + fn from(blobs_bundle: BlobsBundleV1) -> Self { + Self { + commitments: blobs_bundle.commitments, + proofs: blobs_bundle.proofs, + blobs: blobs_bundle.blobs, + } + } +} +impl From> for BlobsBundleV1 { + fn from(json_blobs_bundle: JsonBlobsBundleV1) -> Self { + Self { + commitments: json_blobs_bundle.commitments, + proofs: json_blobs_bundle.proofs, + blobs: json_blobs_bundle.blobs, + } + } } #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index adf5e059b28..0e1fddfad16 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -45,12 +45,12 @@ use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; use types::consts::deneb::BLOB_TX_TYPE; use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlobTransaction}; -use types::Withdrawals; use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, ForkName, }; +use types::{KzgProofs, Withdrawals}; use types::{ ProposerPreparationData, PublicKeyBytes, Signature, SignedBeaconBlock, Slot, Transaction, Uint256, @@ -141,22 +141,53 @@ pub enum BlockProposalContents> { block_value: Uint256, kzg_commitments: KzgCommitments, blobs: Blobs, + proofs: KzgProofs, }, } +impl> From> + for BlockProposalContents +{ + fn from(response: GetPayloadResponse) -> Self { + let (execution_payload, block_value, maybe_bundle) = response.into(); + match maybe_bundle { + Some(bundle) => Self::PayloadAndBlobs { + payload: execution_payload.into(), + block_value, + kzg_commitments: bundle.commitments, + blobs: bundle.blobs, + proofs: bundle.proofs, + }, + None => Self::Payload { + payload: execution_payload.into(), + block_value, + }, + } + } +} + +#[allow(clippy::type_complexity)] impl> BlockProposalContents { - pub fn deconstruct(self) -> (Payload, Option>, Option>) { + pub fn deconstruct( + self, + ) -> ( + Payload, + Option>, + Option>, + Option>, + ) { match self { Self::Payload { payload, block_value: _, - } => (payload, None, None), + } => (payload, None, None, None), Self::PayloadAndBlobs { payload, block_value: _, kzg_commitments, blobs, - } => (payload, Some(kzg_commitments), Some(blobs)), + proofs, + } => (payload, Some(kzg_commitments), Some(blobs), Some(proofs)), } } @@ -171,6 +202,7 @@ impl> BlockProposalContents payload, } } @@ -185,6 +217,7 @@ impl> BlockProposalContents payload, } } @@ -199,6 +232,7 @@ impl> BlockProposalContents block_value, } } @@ -215,6 +249,7 @@ impl> BlockProposalContents ExecutionLayer { } }; - let blob_fut = async { - match current_fork { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { - None - } - ForkName::Deneb => { - debug!( - self.log(), - "Issuing engine_getBlobsBundle"; - "suggested_fee_recipient" => ?payload_attributes.suggested_fee_recipient(), - "prev_randao" => ?payload_attributes.prev_randao(), - "timestamp" => payload_attributes.timestamp(), - "parent_hash" => ?parent_hash, - ); - Some(engine.api.get_blobs_bundle_v1::(payload_id).await) - } - } - }; - let payload_fut = async { + let payload_response = async { debug!( self.log(), "Issuing engine_getPayload"; @@ -1144,45 +1161,30 @@ impl ExecutionLayer { "parent_hash" => ?parent_hash, ); engine.api.get_payload::(current_fork, payload_id).await - }; - let (blob, payload_response) = tokio::join!(blob_fut, payload_fut); - let (execution_payload, block_value) = payload_response.map(|payload_response| { - if payload_response.execution_payload_ref().fee_recipient() != payload_attributes.suggested_fee_recipient() { - error!( - self.log(), - "Inconsistent fee recipient"; - "msg" => "The fee recipient returned from the Execution Engine differs \ - from the suggested_fee_recipient set on the beacon node. This could \ - indicate that fees are being diverted to another address. Please \ - ensure that the value of suggested_fee_recipient is set correctly and \ - that the Execution Engine is trusted.", - "fee_recipient" => ?payload_response.execution_payload_ref().fee_recipient(), - "suggested_fee_recipient" => ?payload_attributes.suggested_fee_recipient(), - ); - } - if f(self, payload_response.execution_payload_ref()).is_some() { - warn!( - self.log(), - "Duplicate payload cached, this might indicate redundant proposal \ - attempts." - ); - } - payload_response.into() - })?; - if let Some(blob) = blob.transpose()? { - // FIXME(sean) cache blobs - Ok(BlockProposalContents::PayloadAndBlobs { - payload: execution_payload.into(), - block_value, - blobs: blob.blobs, - kzg_commitments: blob.kzgs, - }) - } else { - Ok(BlockProposalContents::Payload { - payload: execution_payload.into(), - block_value, - }) + }.await?; + + if payload_response.execution_payload_ref().fee_recipient() != payload_attributes.suggested_fee_recipient() { + error!( + self.log(), + "Inconsistent fee recipient"; + "msg" => "The fee recipient returned from the Execution Engine differs \ + from the suggested_fee_recipient set on the beacon node. This could \ + indicate that fees are being diverted to another address. Please \ + ensure that the value of suggested_fee_recipient is set correctly and \ + that the Execution Engine is trusted.", + "fee_recipient" => ?payload_response.execution_payload_ref().fee_recipient(), + "suggested_fee_recipient" => ?payload_attributes.suggested_fee_recipient(), + ); + } + if f(self, payload_response.execution_payload_ref()).is_some() { + warn!( + self.log(), + "Duplicate payload cached, this might indicate redundant proposal \ + attempts." + ); } + + Ok(payload_response.into()) }) .await .map_err(Box::new) diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index e5a5c70d7d7..773c3fe9d4e 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -6,15 +6,21 @@ use crate::{ }, ExecutionBlock, PayloadAttributes, PayloadId, PayloadStatusV1, PayloadStatusV1Status, }, - ExecutionBlockWithTransactions, + BlobsBundleV1, ExecutionBlockWithTransactions, }; +use kzg::{Kzg, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT, FIELD_ELEMENTS_PER_BLOB}; +use rand::RngCore; use serde::{Deserialize, Serialize}; +use ssz::Encode; use std::collections::HashMap; +use std::sync::Arc; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; +use types::transaction::{BlobTransaction, EcdsaSignature, SignedBlobTransaction}; use types::{ - EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, - ExecutionPayloadMerge, ForkName, Hash256, Uint256, + Blob, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, + ExecutionPayloadDeneb, ExecutionPayloadMerge, ForkName, Hash256, Transaction, Transactions, + Uint256, }; const GAS_LIMIT: u64 = 16384; @@ -119,6 +125,11 @@ pub struct ExecutionBlockGenerator { */ pub shanghai_time: Option, // withdrawals pub deneb_time: Option, // 4844 + /* + * deneb stuff + */ + pub blobs_bundles: HashMap>, + pub kzg: Option>, } impl ExecutionBlockGenerator { @@ -128,6 +139,7 @@ impl ExecutionBlockGenerator { terminal_block_hash: ExecutionBlockHash, shanghai_time: Option, deneb_time: Option, + kzg: Option, ) -> Self { let mut gen = Self { head_block: <_>::default(), @@ -142,6 +154,8 @@ impl ExecutionBlockGenerator { payload_ids: <_>::default(), shanghai_time, deneb_time, + blobs_bundles: <_>::default(), + kzg: kzg.map(Arc::new), }; gen.insert_pow_block(0).unwrap(); @@ -394,6 +408,11 @@ impl ExecutionBlockGenerator { self.payload_ids.get(id).cloned() } + pub fn get_blobs_bundle(&mut self, id: &PayloadId) -> Option> { + // remove it to free memory + self.blobs_bundles.remove(id) + } + pub fn new_payload(&mut self, payload: ExecutionPayload) -> PayloadStatusV1 { let parent = if let Some(parent) = self.blocks.get(&payload.parent_hash()) { parent @@ -561,6 +580,22 @@ impl ExecutionBlockGenerator { } }; + match execution_payload.fork_name() { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => {} + ForkName::Deneb => { + // get random number between 0 and Max Blobs + let num_blobs = rand::random::() % T::max_blobs_per_block(); + let (bundle, transactions) = self.generate_random_blobs(num_blobs)?; + for tx in Vec::from(transactions) { + execution_payload + .transactions_mut() + .push(tx) + .map_err(|_| "transactions are full".to_string())?; + } + self.blobs_bundles.insert(id, bundle); + } + } + *execution_payload.block_hash_mut() = ExecutionBlockHash::from_root(execution_payload.tree_hash_root()); @@ -590,6 +625,88 @@ impl ExecutionBlockGenerator { payload_id: id.map(Into::into), }) } + + fn generate_random_blobs( + &self, + n_blobs: usize, + ) -> Result<(BlobsBundleV1, Transactions), String> { + let mut bundle = BlobsBundleV1::::default(); + let mut transactions = vec![]; + for blob_index in 0..n_blobs { + // fill a vector with random bytes + let mut blob_bytes = [0u8; BYTES_PER_BLOB]; + rand::thread_rng().fill_bytes(&mut blob_bytes); + // Ensure that the blob is canonical by ensuring that + // each field element contained in the blob is < BLS_MODULUS + for i in 0..FIELD_ELEMENTS_PER_BLOB { + blob_bytes[i * BYTES_PER_FIELD_ELEMENT + BYTES_PER_FIELD_ELEMENT - 1] = 0; + } + + let blob = Blob::::new(Vec::from(blob_bytes)) + .map_err(|e| format!("error constructing random blob: {:?}", e))?; + + let commitment = self + .kzg + .as_ref() + .ok_or("kzg not initialized")? + .blob_to_kzg_commitment(blob_bytes.into()) + .map_err(|e| format!("error computing kzg commitment: {:?}", e))?; + + let proof = self + .kzg + .as_ref() + .ok_or("kzg not initialized")? + .compute_blob_kzg_proof(blob_bytes.into(), commitment) + .map_err(|e| format!("error computing kzg proof: {:?}", e))?; + + let versioned_hash = commitment.calculate_versioned_hash(); + + let blob_transaction = BlobTransaction { + chain_id: Default::default(), + nonce: 0, + max_priority_fee_per_gas: Default::default(), + max_fee_per_gas: Default::default(), + gas: 100000, + to: None, + value: Default::default(), + data: Default::default(), + access_list: Default::default(), + max_fee_per_data_gas: Default::default(), + versioned_hashes: vec![versioned_hash].into(), + }; + let bad_signature = EcdsaSignature { + y_parity: false, + r: Uint256::from(0), + s: Uint256::from(0), + }; + let signed_blob_transaction = SignedBlobTransaction { + message: blob_transaction, + signature: bad_signature, + }; + // calculate transaction bytes + let tx_bytes = [0x05u8] + .into_iter() + .chain(signed_blob_transaction.as_ssz_bytes().into_iter()) + .collect::>(); + let tx = Transaction::::from(tx_bytes); + + transactions.push(tx); + bundle + .blobs + .push(blob) + .map_err(|_| format!("blobs are full, blob index: {:?}", blob_index))?; + bundle + .commitments + .push(commitment) + .map_err(|_| format!("blobs are full, blob index: {:?}", blob_index))?; + bundle + .proofs + .push(proof) + .map_err(|_| format!("blobs are full, blob index: {:?}", blob_index))?; + } + + Ok((bundle, transactions.into())) + } } fn payload_id_from_u64(n: u64) -> PayloadId { @@ -650,6 +767,7 @@ mod test { ExecutionBlockHash::zero(), None, None, + None, ); for i in 0..=TERMINAL_BLOCK { diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index aae1a0b8989..6122f28dce7 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -224,6 +224,8 @@ pub async fn handle_rpc( ) })?; + let maybe_blobs = ctx.execution_block_generator.write().get_blobs_bundle(&id); + // validate method called correctly according to shanghai fork time if ctx .execution_block_generator @@ -291,6 +293,12 @@ pub async fn handle_rpc( serde_json::to_value(JsonGetPayloadResponseV3 { execution_payload, block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), + blobs_bundle: maybe_blobs + .ok_or(( + "No blobs returned despite V3 Payload".to_string(), + GENERIC_ERROR_CODE, + ))? + .into(), }) .unwrap() } @@ -324,7 +332,7 @@ pub async fn handle_rpc( .map(|opt| opt.map(JsonPayloadAttributes::V1)) .transpose() } - ForkName::Capella => { + ForkName::Capella | ForkName::Deneb => { get_param::>(params, 1) .map(|opt| opt.map(JsonPayloadAttributes::V2)) .transpose() diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 44fc2a5ec2d..0c6f5ce666e 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -5,6 +5,7 @@ use crate::{ }, Config, *, }; +use kzg::Kzg; use sensitive_url::SensitiveUrl; use task_executor::TaskExecutor; use tempfile::NamedTempFile; @@ -33,6 +34,7 @@ impl MockExecutionLayer { Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()), spec, None, + None, ) } @@ -46,6 +48,7 @@ impl MockExecutionLayer { jwt_key: Option, spec: ChainSpec, builder_url: Option, + kzg: Option, ) -> Self { let handle = executor.handle().unwrap(); @@ -58,6 +61,7 @@ impl MockExecutionLayer { spec.terminal_block_hash, shanghai_time, deneb_time, + kzg, ); let url = SensitiveUrl::parse(&server.url()).unwrap(); diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 60f5bf341f6..ef728722da4 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -8,6 +8,7 @@ use bytes::Bytes; use environment::null_logger; use execution_block_generator::PoWBlock; use handle_rpc::handle_rpc; +use kzg::Kzg; use parking_lot::{Mutex, RwLock, RwLockWriteGuard}; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -96,10 +97,15 @@ impl MockServer { ExecutionBlockHash::zero(), None, // FIXME(capella): should this be the default? None, // FIXME(deneb): should this be the default? + None, // FIXME(deneb): should this be the default? ) } - pub fn new_with_config(handle: &runtime::Handle, config: MockExecutionConfig) -> Self { + pub fn new_with_config( + handle: &runtime::Handle, + config: MockExecutionConfig, + kzg: Option, + ) -> Self { let MockExecutionConfig { jwt_key, terminal_difficulty, @@ -117,6 +123,7 @@ impl MockServer { terminal_block_hash, shanghai_time, deneb_time, + kzg, ); let ctx: Arc> = Arc::new(Context { @@ -168,6 +175,7 @@ impl MockServer { *self.ctx.engine_capabilities.write() = engine_capabilities; } + #[allow(clippy::too_many_arguments)] pub fn new( handle: &runtime::Handle, jwt_key: JwtKey, @@ -176,6 +184,7 @@ impl MockServer { terminal_block_hash: ExecutionBlockHash, shanghai_time: Option, deneb_time: Option, + kzg: Option, ) -> Self { Self::new_with_config( handle, @@ -188,6 +197,7 @@ impl MockServer { shanghai_time, deneb_time, }, + kzg, ) } diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index b3e5bf39cc7..d122743089c 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -480,7 +480,7 @@ pub async fn proposer_boost_re_org_test( // Produce block B and process it halfway through the slot. let (block_b, mut state_b) = harness.make_block(state_a.clone(), slot_b).await; - let block_b_root = block_b.canonical_root(); + let block_b_root = block_b.0.canonical_root(); let obs_time = slot_clock.start_of(slot_b).unwrap() + slot_clock.slot_duration() / 2; slot_clock.set_current_time(obs_time); @@ -573,8 +573,18 @@ pub async fn proposer_boost_re_org_test( // Check the fork choice updates that were sent. let forkchoice_updates = forkchoice_updates.lock(); - let block_a_exec_hash = block_a.message().execution_payload().unwrap().block_hash(); - let block_b_exec_hash = block_b.message().execution_payload().unwrap().block_hash(); + let block_a_exec_hash = block_a + .0 + .message() + .execution_payload() + .unwrap() + .block_hash(); + let block_b_exec_hash = block_b + .0 + .message() + .execution_payload() + .unwrap() + .block_hash(); let block_c_timestamp = block_c.message().execution_payload().unwrap().timestamp(); @@ -679,7 +689,7 @@ pub async fn fork_choice_before_proposal() { let state_a = harness.get_current_state(); let (block_b, state_b) = harness.make_block(state_a.clone(), slot_b).await; let block_root_b = harness - .process_block(slot_b, block_b.canonical_root(), block_b) + .process_block(slot_b, block_b.0.canonical_root(), block_b) .await .unwrap(); @@ -694,7 +704,7 @@ pub async fn fork_choice_before_proposal() { let (block_c, state_c) = harness.make_block(state_a, slot_c).await; let block_root_c = harness - .process_block(slot_c, block_c.canonical_root(), block_c.clone()) + .process_block(slot_c, block_c.0.canonical_root(), block_c.clone()) .await .unwrap(); diff --git a/beacon_node/network/src/beacon_processor/tests.rs b/beacon_node/network/src/beacon_processor/tests.rs index 30e7e358c02..0f434fdc375 100644 --- a/beacon_node/network/src/beacon_processor/tests.rs +++ b/beacon_node/network/src/beacon_processor/tests.rs @@ -107,7 +107,7 @@ impl TestRig { "precondition: current slot is one after head" ); - let (next_block, next_state) = harness + let (next_block_tuple, next_state) = harness .make_block(head.beacon_state.clone(), harness.chain.slot().unwrap()) .await; @@ -133,9 +133,9 @@ impl TestRig { .get_unaggregated_attestations( &AttestationStrategy::AllValidators, &next_state, - next_block.state_root(), - next_block.canonical_root(), - next_block.slot(), + next_block_tuple.0.state_root(), + next_block_tuple.0.canonical_root(), + next_block_tuple.0.slot(), ) .into_iter() .flatten() @@ -145,9 +145,9 @@ impl TestRig { .make_attestations( &harness.get_all_validators(), &next_state, - next_block.state_root(), - next_block.canonical_root().into(), - next_block.slot(), + next_block_tuple.0.state_root(), + next_block_tuple.0.canonical_root().into(), + next_block_tuple.0.slot(), ) .into_iter() .filter_map(|(_, aggregate_opt)| aggregate_opt) @@ -209,7 +209,7 @@ impl TestRig { Self { chain, - next_block: Arc::new(next_block), + next_block: Arc::new(next_block_tuple.0), attestations, next_block_attestations, next_block_aggregate_attestations, diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index 82bf642f180..e596fb2e155 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -179,15 +179,15 @@ impl ForkChoiceTest { let slot = self.harness.get_current_slot(); let (block, state_) = self.harness.make_block(state, slot).await; state = state_; - if !predicate(block.message(), &state) { + if !predicate(block.0.message(), &state) { break; } if let Ok(block_hash) = self.harness.process_block_result(block.clone()).await { self.harness.attest_block( &state, - block.state_root(), + block.0.state_root(), block_hash, - &block, + &block.0, &validators, ); self.harness.advance_slot(); @@ -273,8 +273,8 @@ impl ForkChoiceTest { ) .unwrap(); let slot = self.harness.get_current_slot(); - let (mut signed_block, mut state) = self.harness.make_block(state, slot).await; - func(&mut signed_block, &mut state); + let (mut block_tuple, mut state) = self.harness.make_block(state, slot).await; + func(&mut block_tuple.0, &mut state); let current_slot = self.harness.get_current_slot(); self.harness .chain @@ -282,8 +282,8 @@ impl ForkChoiceTest { .fork_choice_write_lock() .on_block( current_slot, - signed_block.message(), - signed_block.canonical_root(), + block_tuple.0.message(), + block_tuple.0.canonical_root(), Duration::from_secs(0), &state, PayloadVerificationStatus::Verified, @@ -315,8 +315,8 @@ impl ForkChoiceTest { ) .unwrap(); let slot = self.harness.get_current_slot(); - let (mut signed_block, mut state) = self.harness.make_block(state, slot).await; - mutation_func(&mut signed_block, &mut state); + let (mut block_tuple, mut state) = self.harness.make_block(state, slot).await; + mutation_func(&mut block_tuple.0, &mut state); let current_slot = self.harness.get_current_slot(); let err = self .harness @@ -325,8 +325,8 @@ impl ForkChoiceTest { .fork_choice_write_lock() .on_block( current_slot, - signed_block.message(), - signed_block.canonical_root(), + block_tuple.0.message(), + block_tuple.0.canonical_root(), Duration::from_secs(0), &state, PayloadVerificationStatus::Verified, diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index fde54bc721c..f0a1f0ee4e5 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -1,5 +1,6 @@ use crate::test_utils::TestRandom; -use crate::{Blob, EthSpec, Hash256, SignedRoot, Slot}; +use crate::{Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, SignedBlobSidecar, SignedRoot, Slot}; +use bls::SecretKey; use derivative::Derivative; use kzg::{KzgCommitment, KzgProof}; use serde_derive::{Deserialize, Serialize}; @@ -72,7 +73,7 @@ impl Ord for BlobSidecar { } pub type BlobSidecarList = VariableList>, ::MaxBlobsPerBlock>; -pub type Blobs = VariableList, ::MaxExtraDataBytes>; +pub type Blobs = VariableList, ::MaxBlobsPerBlock>; impl SignedRoot for BlobSidecar {} @@ -93,4 +94,28 @@ impl BlobSidecar { // Fixed part Self::empty().as_ssz_bytes().len() } + + // this is mostly not used except for in testing + pub fn sign( + self: Arc, + secret_key: &SecretKey, + fork: &Fork, + genesis_validators_root: Hash256, + spec: &ChainSpec, + ) -> SignedBlobSidecar { + let signing_epoch = self.slot.epoch(T::slots_per_epoch()); + let domain = spec.get_domain( + signing_epoch, + Domain::BlobSidecar, + fork, + genesis_validators_root, + ); + let message = self.signing_root(domain); + let signature = secret_key.sign(message); + + SignedBlobSidecar { + message: self, + signature, + } + } } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index d38d7f368b3..617cbcaf02d 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -204,6 +204,7 @@ pub type Address = H160; pub type ForkVersion = [u8; 4]; pub type BLSFieldElement = Uint256; pub type Blob = FixedVector::BytesPerBlob>; +pub type KzgProofs = VariableList::MaxBlobsPerBlock>; pub type VersionedHash = Hash256; pub type Hash64 = ethereum_types::H64; diff --git a/crypto/kzg/src/kzg_commitment.rs b/crypto/kzg/src/kzg_commitment.rs index 0f2725b7522..267f704629c 100644 --- a/crypto/kzg/src/kzg_commitment.rs +++ b/crypto/kzg/src/kzg_commitment.rs @@ -1,18 +1,29 @@ use c_kzg::{Bytes48, BYTES_PER_COMMITMENT}; use derivative::Derivative; +use eth2_hashing::hash_fixed; use serde::de::{Deserialize, Deserializer}; use serde::ser::{Serialize, Serializer}; use ssz_derive::{Decode, Encode}; use std::fmt; use std::fmt::{Debug, Display, Formatter}; use std::str::FromStr; -use tree_hash::{PackedEncoding, TreeHash}; +use tree_hash::{Hash256, PackedEncoding, TreeHash}; + +pub const BLOB_COMMITMENT_VERSION_KZG: u8 = 0x01; #[derive(Derivative, Clone, Copy, Encode, Decode)] #[derivative(PartialEq, Eq, Hash)] #[ssz(struct_behaviour = "transparent")] pub struct KzgCommitment(pub [u8; BYTES_PER_COMMITMENT]); +impl KzgCommitment { + pub fn calculate_versioned_hash(&self) -> Hash256 { + let mut versioned_hash = hash_fixed(&self.0); + versioned_hash[0] = BLOB_COMMITMENT_VERSION_KZG; + Hash256::from_slice(versioned_hash.as_slice()) + } +} + impl From for Bytes48 { fn from(value: KzgCommitment) -> Self { value.0.into() diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index 5379d36ed0a..b300e2d3bb2 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -20,6 +20,7 @@ pub enum Error { } /// A wrapper over a kzg library that holds the trusted setup parameters. +#[derive(Debug)] pub struct Kzg { trusted_setup: KzgSettings, } diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index d4fd115bec3..1b822a322ca 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -236,7 +236,7 @@ impl LocalExecutionNode { panic!("Failed to write jwt file {}", e); } Self { - server: MockServer::new_with_config(&context.executor.handle().unwrap(), config), + server: MockServer::new_with_config(&context.executor.handle().unwrap(), config, None), datadir, } } From e5343916e082e9d984393cc3a5532f3fac5e0341 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Mon, 1 May 2023 10:10:14 +0200 Subject: [PATCH 83/96] minor fix --- beacon_node/execution_layer/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 61430350fec..15ad7764c55 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1709,6 +1709,7 @@ impl ExecutionLayer { ForkName::Merge => ExecutionPayloadMerge::default().into(), ForkName::Capella => ExecutionPayloadCapella::default().into(), ForkName::Eip4844 => ExecutionPayloadEip4844::default().into(), + ForkName::Eip6110 => ExecutionPayloadEip6110::default().into(), ForkName::Base | ForkName::Altair => { return Err(Error::InvalidForkForPayload); } From 8068c14528c8758a926d15a04ba706e3cde9ab1e Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Tue, 2 May 2023 13:06:43 +0200 Subject: [PATCH 84/96] fix `config.yaml` --- .../built_in_network_configs/eip4844/config.yaml | 4 ---- .../built_in_network_configs/gnosis/config.yaml | 3 --- .../built_in_network_configs/sepolia/config.yaml | 4 ---- 3 files changed, 11 deletions(-) diff --git a/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml b/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml index ecd9ad80cc8..f7334b6187c 100644 --- a/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/eip4844/config.yaml @@ -38,10 +38,6 @@ CAPELLA_FORK_EPOCH: 1 EIP4844_FORK_VERSION: 0x50484404 EIP4844_FORK_EPOCH: 5 -# EIP6110 -EIP6110_FORK_VERSION: 0x60484404 -EIP6110_FORK_EPOCH: 10 - # Time parameters # --------------------------------------------------------------- # 12 seconds diff --git a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml index 0386babb800..6aa2c9590a5 100644 --- a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml @@ -42,9 +42,6 @@ CAPELLA_FORK_EPOCH: 18446744073709551615 # Eip4844 EIP4844_FORK_VERSION: 0x04000064 EIP4844_FORK_EPOCH: 18446744073709551615 -# Eip6110 -EIP6110_FORK_VERSION: 0x05000064 -EIP6110_FORK_EPOCH: 18446744073709551615 # Sharding SHARDING_FORK_VERSION: 0x03000064 SHARDING_FORK_EPOCH: 18446744073709551615 diff --git a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml index 95a2eaf0d28..4ba006ec945 100644 --- a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml @@ -36,10 +36,6 @@ CAPELLA_FORK_EPOCH: 56832 EIP4844_FORK_VERSION: 0x03001020 EIP4844_FORK_EPOCH: 18446744073709551615 -# Eip6110 -EIP6110_FORK_VERSION: 0x04001020 -EIP6110_FORK_EPOCH: 18446744073709551615 - # Sharding SHARDING_FORK_VERSION: 0x04001020 SHARDING_FORK_EPOCH: 18446744073709551615 From 9db6b39dc3b267092959a1bb3c2677c895b6fbd6 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 2 May 2023 19:14:02 -0400 Subject: [PATCH 85/96] fix check on max request size (#4250) --- .../beacon_processor/worker/rpc_methods.rs | 33 ++++++------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index c8667b3528f..5f282ecfbee 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -625,7 +625,7 @@ impl Worker { ); // Should not send more than max request blocks - if req.count > MAX_REQUEST_BLOB_SIDECARS { + if req.count * T::EthSpec::max_blobs_per_block() as u64 > MAX_REQUEST_BLOB_SIDECARS { return self.send_error_response( peer_id, RPCResponseErrorCode::InvalidRequest, @@ -808,28 +808,15 @@ impl Worker { .slot() .unwrap_or_else(|_| self.chain.slot_clock.genesis_slot()); - if blobs_sent < (req.count as usize) { - debug!( - self.log, - "BlobsByRange Response processed"; - "peer" => %peer_id, - "msg" => "Failed to return all requested blobs", - "start_slot" => req.start_slot, - "current_slot" => current_slot, - "requested" => req.count, - "returned" => blobs_sent - ); - } else { - debug!( - self.log, - "BlobsByRange Response processed"; - "peer" => %peer_id, - "start_slot" => req.start_slot, - "current_slot" => current_slot, - "requested" => req.count, - "returned" => blobs_sent - ); - } + debug!( + self.log, + "BlobsByRange Response processed"; + "peer" => %peer_id, + "start_slot" => req.start_slot, + "current_slot" => current_slot, + "requested" => req.count, + "returned" => blobs_sent + ); if send_response { // send the stream terminator From 8cb5ce75ea9de307307687f7f54c528d8ed54d79 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Wed, 3 May 2023 10:46:13 +0200 Subject: [PATCH 86/96] add `$h:block` --- consensus/types/src/payload.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 5846ce5fb44..c28e61740dc 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -752,7 +752,8 @@ macro_rules! impl_exec_payload_common { $block_type_variant:ident, // Blinded | Full $is_default_with_empty_roots:block, $f:block, - $g:block) => { + $g:block, + $h:block) => { impl ExecPayload for $wrapper_type { fn block_type() -> BlockType { BlockType::$block_type_variant @@ -812,7 +813,8 @@ macro_rules! impl_exec_payload_common { } fn deposit_receipts(&self) -> Result, Error> { - todo!() + let h = $h; + h(self) } } @@ -851,6 +853,15 @@ macro_rules! impl_exec_payload_for_fork { wrapper_ref_type.withdrawals_root() }; c + }, + { + let c: for<'a> fn( + &'a $wrapper_type_header, + ) -> Result, Error> = |payload: &$wrapper_type_header| { + let wrapper_ref_type = BlindedPayloadRef::$fork_variant(&payload); + wrapper_ref_type.deposit_receipts() + }; + c } ); @@ -930,6 +941,14 @@ macro_rules! impl_exec_payload_for_fork { wrapper_ref_type.withdrawals_root() }; c + }, + { + let c: for<'a> fn(&'a $wrapper_type_full) -> Result, Error> = + |payload: &$wrapper_type_full| { + let wrapper_ref_type = FullPayloadRef::$fork_variant(&payload); + wrapper_ref_type.deposit_receipts() + }; + c } ); From 875d834ce923c42ed384c2a58260a68fad8a5f99 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Wed, 3 May 2023 11:33:16 +0200 Subject: [PATCH 87/96] add `deposit_receipts` to `ExecutionPayloadBodyV1` --- beacon_node/execution_layer/src/engine_api.rs | 7 ++++--- .../execution_layer/src/engine_api/json_structures.rs | 9 +++++++++ beacon_node/execution_layer/src/test_utils/handle_rpc.rs | 3 +++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 1b18108c262..382d91da57a 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -17,9 +17,9 @@ use std::convert::TryFrom; use strum::IntoStaticStr; use superstruct::superstruct; pub use types::{ - Address, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, - ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, - Withdrawal, Withdrawals, + Address, DepositReceipt, DepositReceipts, EthSpec, ExecutionBlockHash, ExecutionPayload, + ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, + Uint256, VariableList, Withdrawal, Withdrawals, }; use types::{ ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadEip6110, @@ -469,6 +469,7 @@ impl GetPayloadResponse { pub struct ExecutionPayloadBodyV1 { pub transactions: Transactions, pub withdrawals: Option>, + pub deposit_receipts: Option>, } impl ExecutionPayloadBodyV1 { diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 4e057a21779..7761a5747fd 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -694,6 +694,7 @@ pub struct JsonExecutionPayloadBodyV1 { #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] pub transactions: Transactions, pub withdrawals: Option>, + pub deposit_receipts: Option>, } impl From> for ExecutionPayloadBodyV1 { @@ -708,6 +709,14 @@ impl From> for ExecutionPayloadBodyV1< .collect::>(), ) }), + deposit_receipts: value.deposit_receipts.map(|json_receipts| { + DepositReceipts::::from( + json_receipts + .into_iter() + .map(Into::into) + .collect::>(), + ) + }), } } } diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 9dcd03a60f4..77806b897af 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -569,6 +569,9 @@ pub async fn handle_rpc( .withdrawals() .ok() .map(|withdrawals| VariableList::from(withdrawals.clone())), + deposit_receipts: block.deposit_receipts().ok().map( + |deposit_receipts| VariableList::from(deposit_receipts.clone()), + ), })); } None => response.push(None), From b9ea79a739790b98f6a93b2f7f4d92c28d61f650 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Wed, 3 May 2023 13:03:05 +0200 Subject: [PATCH 88/96] fix `process_operations` --- .../src/per_block_processing/process_operations.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 0698558b498..558554ed6fb 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -53,6 +53,10 @@ pub fn process_operations>( process_deposits(state, block_body.deposits(), spec)?; process_exits(state, block_body.voluntary_exits(), verify_signatures, spec)?; + if let Ok(bls_to_execution_changes) = block_body.bls_to_execution_changes() { + process_bls_to_execution_changes(state, bls_to_execution_changes, verify_signatures, spec)?; + } + if let Ok(payload) = block_body.execution_payload() { if is_execution_enabled(state, block_body) { let deposit_receipts = payload.deposit_receipts()?; From a6782af3232fec08fef7dd5b500ee33639041293 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Thu, 4 May 2023 20:30:59 +0200 Subject: [PATCH 89/96] `deposit_receipts` -> `deposit_receipts_root` --- beacon_node/execution_layer/src/engine_api.rs | 31 +++- .../process_operations.rs | 2 + consensus/types/src/payload.rs | 138 +++++++----------- 3 files changed, 85 insertions(+), 86 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 382d91da57a..932b86b76d0 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -555,7 +555,36 @@ impl ExecutionPayloadBodyV1 { )) } } - ExecutionPayloadHeader::Eip6110(_) => todo!(), + ExecutionPayloadHeader::Eip6110(header) => { + if let (Some(withdrawals), Some(deposit_receipts)) = + (self.withdrawals, self.deposit_receipts) + { + Ok(ExecutionPayload::Eip6110(ExecutionPayloadEip6110 { + parent_hash: header.parent_hash, + fee_recipient: header.fee_recipient, + state_root: header.state_root, + receipts_root: header.receipts_root, + logs_bloom: header.logs_bloom, + prev_randao: header.prev_randao, + block_number: header.block_number, + gas_limit: header.gas_limit, + gas_used: header.gas_used, + timestamp: header.timestamp, + extra_data: header.extra_data, + base_fee_per_gas: header.base_fee_per_gas, + excess_data_gas: header.excess_data_gas, + block_hash: header.block_hash, + transactions: self.transactions, + withdrawals, + deposit_receipts, + })) + } else { + Err(format!( + "block {} is eip6110 but payload body doesn't have withdrawals and deposit_receipts", + header.block_hash + )) + } + } } } } diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 558554ed6fb..92237b4083b 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -57,6 +57,7 @@ pub fn process_operations>( process_bls_to_execution_changes(state, bls_to_execution_changes, verify_signatures, spec)?; } + /* if let Ok(payload) = block_body.execution_payload() { if is_execution_enabled(state, block_body) { let deposit_receipts = payload.deposit_receipts()?; @@ -65,6 +66,7 @@ pub fn process_operations>( } } } + */ Ok(()) } diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index c28e61740dc..6c948bb5b04 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -39,7 +39,7 @@ pub trait ExecPayload: Debug + Clone + PartialEq + Hash + TreeHash + fn transactions(&self) -> Option<&Transactions>; /// fork-specific fields fn withdrawals_root(&self) -> Result; - fn deposit_receipts(&self) -> Result, Error>; + fn deposit_receipts_root(&self) -> Result; /// Is this a default payload with 0x0 roots for transactions and withdrawals? fn is_default_with_zero_roots(&self) -> bool; @@ -268,6 +268,17 @@ impl ExecPayload for FullPayload { } } + fn deposit_receipts_root(&self) -> Result { + match self { + FullPayload::Merge(_) => Err(Error::IncorrectStateVariant), + FullPayload::Capella(_) => Err(Error::IncorrectStateVariant), + FullPayload::Eip4844(_) => Err(Error::IncorrectStateVariant), + FullPayload::Eip6110(ref inner) => { + Ok(inner.execution_payload.deposit_receipts.tree_hash_root()) + } + } + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_full_payload_ref!(&'a _, self.to_ref(), move |payload, cons| { cons(payload); @@ -279,24 +290,6 @@ impl ExecPayload for FullPayload { // For full payloads the empty/zero distinction does not exist. self.is_default_with_zero_roots() } - - fn deposit_receipts(&self) -> Result, Error> { - match self { - FullPayload::Merge(_) => { - // Return an "empty" or "default" value for the Merge variant - Ok(Vec::new().into()) - } - FullPayload::Capella(_) => { - // Return an "empty" or "default" value for the Capella variant - Ok(Vec::new().into()) - } - FullPayload::Eip4844(_) => { - // Return an "empty" or "default" value for the EIP-4844 variant - Ok(Vec::new().into()) - } - FullPayload::Eip6110(ref inner) => Ok(inner.execution_payload.deposit_receipts.clone()), - } - } } impl FullPayload { @@ -398,6 +391,17 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { } } + fn deposit_receipts_root(&self) -> Result { + match self { + FullPayloadRef::Merge(_) => Err(Error::IncorrectStateVariant), + FullPayloadRef::Capella(_) => Err(Error::IncorrectStateVariant), + FullPayloadRef::Eip4844(_) => Err(Error::IncorrectStateVariant), + FullPayloadRef::Eip6110(inner) => { + Ok(inner.execution_payload.deposit_receipts.tree_hash_root()) + } + } + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_full_payload_ref!(&'a _, self, move |payload, cons| { cons(payload); @@ -409,24 +413,6 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { // For full payloads the empty/zero distinction does not exist. self.is_default_with_zero_roots() } - - fn deposit_receipts(&self) -> Result, Error> { - match self { - FullPayloadRef::Merge(_) => { - // Return an "empty" or "default" value for the Merge variant - Ok(Vec::new().into()) - } - FullPayloadRef::Capella(_) => { - // Return an "empty" or "default" value for the Capella variant - Ok(Vec::new().into()) - } - FullPayloadRef::Eip4844(_) => { - // Return an "empty" or "default" value for the EIP-4844 variant - Ok(Vec::new().into()) - } - FullPayloadRef::Eip6110(inner) => Ok(inner.execution_payload.deposit_receipts.clone()), - } - } } impl AbstractExecPayload for FullPayload { @@ -595,6 +581,17 @@ impl ExecPayload for BlindedPayload { } } + fn deposit_receipts_root(&self) -> Result { + match self { + BlindedPayload::Merge(_) => Err(Error::IncorrectStateVariant), + BlindedPayload::Capella(_) => Err(Error::IncorrectStateVariant), + BlindedPayload::Eip4844(_) => Err(Error::IncorrectStateVariant), + BlindedPayload::Eip6110(ref inner) => { + Ok(inner.execution_payload_header.deposit_receipts_root) + } + } + } + fn is_default_with_zero_roots(&self) -> bool { self.to_ref().is_default_with_zero_roots() } @@ -606,24 +603,6 @@ impl ExecPayload for BlindedPayload { fn is_default_with_empty_roots(&self) -> bool { self.to_ref().is_default_with_empty_roots() } - - fn deposit_receipts(&self) -> Result, Error> { - match self { - BlindedPayload::Merge(_) => { - // Return an "empty" or "default" value for the Merge variant - Ok(Vec::new().into()) - } - BlindedPayload::Capella(_) => { - // Return an "empty" or "default" value for the Capella variant - Ok(Vec::new().into()) - } - BlindedPayload::Eip4844(_) => { - // Return an "empty" or "default" value for the EIP-4844 variant - Ok(Vec::new().into()) - } - BlindedPayload::Eip6110(ref inner) => Ok(inner.deposit_receipts()?), - } - } } impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { @@ -706,6 +685,17 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { } } + fn deposit_receipts_root(&self) -> Result { + match self { + BlindedPayloadRef::Merge(_) => Err(Error::IncorrectStateVariant), + BlindedPayloadRef::Capella(_) => Err(Error::IncorrectStateVariant), + BlindedPayloadRef::Eip4844(_) => Err(Error::IncorrectStateVariant), + BlindedPayloadRef::Eip6110(inner) => { + Ok(inner.execution_payload_header.deposit_receipts_root) + } + } + } + fn is_default_with_zero_roots<'a>(&'a self) -> bool { map_blinded_payload_ref!(&'b _, self, move |payload, cons| { cons(payload); @@ -719,27 +709,6 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { payload.is_default_with_empty_roots() }) } - - fn deposit_receipts(&self) -> Result, Error> { - match self { - BlindedPayloadRef::Merge(_) => { - // Return an "empty" or "default" value for the Merge variant - Ok(Vec::new().into()) - } - BlindedPayloadRef::Capella(_) => { - // Return an "empty" or "default" value for the Capella variant - Ok(Vec::new().into()) - } - BlindedPayloadRef::Eip4844(_) => { - // Return an "empty" or "default" value for the Eip4844 variant - Ok(Vec::new().into()) - } - BlindedPayloadRef::Eip6110(_) => { - // Return an "empty" or "default" value for the Eip6110 variant - Ok(Vec::new().into()) - } - } - } } macro_rules! impl_exec_payload_common { @@ -812,7 +781,7 @@ macro_rules! impl_exec_payload_common { g(self) } - fn deposit_receipts(&self) -> Result, Error> { + fn deposit_receipts_root(&self) -> Result { let h = $h; h(self) } @@ -855,12 +824,11 @@ macro_rules! impl_exec_payload_for_fork { c }, { - let c: for<'a> fn( - &'a $wrapper_type_header, - ) -> Result, Error> = |payload: &$wrapper_type_header| { - let wrapper_ref_type = BlindedPayloadRef::$fork_variant(&payload); - wrapper_ref_type.deposit_receipts() - }; + let c: for<'a> fn(&'a $wrapper_type_header) -> Result = + |payload: &$wrapper_type_header| { + let wrapper_ref_type = BlindedPayloadRef::$fork_variant(&payload); + wrapper_ref_type.deposit_receipts_root() + }; c } ); @@ -943,10 +911,10 @@ macro_rules! impl_exec_payload_for_fork { c }, { - let c: for<'a> fn(&'a $wrapper_type_full) -> Result, Error> = + let c: for<'a> fn(&'a $wrapper_type_full) -> Result = |payload: &$wrapper_type_full| { let wrapper_ref_type = FullPayloadRef::$fork_variant(&payload); - wrapper_ref_type.deposit_receipts() + wrapper_ref_type.deposit_receipts_root() }; c } From b451305ed1f2c80859a81a3887f40f952bb60102 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Mon, 8 May 2023 10:44:06 +0200 Subject: [PATCH 90/96] `deposit_receipts_root` -> `deposit_receipts` --- .../process_operations.rs | 2 - .../types/src/execution_payload_header.rs | 6 +-- consensus/types/src/payload.rs | 39 +++++++++---------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 92237b4083b..558554ed6fb 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -57,7 +57,6 @@ pub fn process_operations>( process_bls_to_execution_changes(state, bls_to_execution_changes, verify_signatures, spec)?; } - /* if let Ok(payload) = block_body.execution_payload() { if is_execution_enabled(state, block_body) { let deposit_receipts = payload.deposit_receipts()?; @@ -66,7 +65,6 @@ pub fn process_operations>( } } } - */ Ok(()) } diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index dd7b879fb8b..14a8f28d03b 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -83,7 +83,7 @@ pub struct ExecutionPayloadHeader { pub withdrawals_root: Hash256, #[superstruct(only(Eip6110))] #[superstruct(getter(copy))] - pub deposit_receipts_root: Hash256, + pub deposit_receipts: DepositReceipts, } impl ExecutionPayloadHeader { @@ -184,7 +184,7 @@ impl ExecutionPayloadHeaderEip4844 { block_hash: self.block_hash, transactions_root: self.transactions_root, withdrawals_root: self.withdrawals_root, - deposit_receipts_root: Hash256::zero(), + deposit_receipts: DepositReceipts::::default(), } } } @@ -273,7 +273,7 @@ impl<'a, T: EthSpec> From<&'a ExecutionPayloadEip6110> for ExecutionPayloadHe block_hash: payload.block_hash, transactions_root: payload.transactions.tree_hash_root(), withdrawals_root: payload.withdrawals.tree_hash_root(), - deposit_receipts_root: payload.deposit_receipts.tree_hash_root(), + deposit_receipts: payload.deposit_receipts.clone(), } } } diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 6c948bb5b04..c1f31cb6155 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -39,7 +39,7 @@ pub trait ExecPayload: Debug + Clone + PartialEq + Hash + TreeHash + fn transactions(&self) -> Option<&Transactions>; /// fork-specific fields fn withdrawals_root(&self) -> Result; - fn deposit_receipts_root(&self) -> Result; + fn deposit_receipts(&self) -> Result, Error>; /// Is this a default payload with 0x0 roots for transactions and withdrawals? fn is_default_with_zero_roots(&self) -> bool; @@ -268,14 +268,12 @@ impl ExecPayload for FullPayload { } } - fn deposit_receipts_root(&self) -> Result { + fn deposit_receipts(&self) -> Result, Error> { match self { FullPayload::Merge(_) => Err(Error::IncorrectStateVariant), FullPayload::Capella(_) => Err(Error::IncorrectStateVariant), FullPayload::Eip4844(_) => Err(Error::IncorrectStateVariant), - FullPayload::Eip6110(ref inner) => { - Ok(inner.execution_payload.deposit_receipts.tree_hash_root()) - } + FullPayload::Eip6110(ref inner) => Ok(inner.execution_payload.deposit_receipts.clone()), } } @@ -391,14 +389,12 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { } } - fn deposit_receipts_root(&self) -> Result { + fn deposit_receipts(&self) -> Result, Error> { match self { FullPayloadRef::Merge(_) => Err(Error::IncorrectStateVariant), FullPayloadRef::Capella(_) => Err(Error::IncorrectStateVariant), FullPayloadRef::Eip4844(_) => Err(Error::IncorrectStateVariant), - FullPayloadRef::Eip6110(inner) => { - Ok(inner.execution_payload.deposit_receipts.tree_hash_root()) - } + FullPayloadRef::Eip6110(inner) => Ok(inner.execution_payload.deposit_receipts.clone()), } } @@ -581,13 +577,13 @@ impl ExecPayload for BlindedPayload { } } - fn deposit_receipts_root(&self) -> Result { + fn deposit_receipts(&self) -> Result, Error> { match self { BlindedPayload::Merge(_) => Err(Error::IncorrectStateVariant), BlindedPayload::Capella(_) => Err(Error::IncorrectStateVariant), BlindedPayload::Eip4844(_) => Err(Error::IncorrectStateVariant), BlindedPayload::Eip6110(ref inner) => { - Ok(inner.execution_payload_header.deposit_receipts_root) + Ok(inner.execution_payload_header.deposit_receipts.clone()) } } } @@ -685,13 +681,13 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { } } - fn deposit_receipts_root(&self) -> Result { + fn deposit_receipts(&self) -> Result, Error> { match self { BlindedPayloadRef::Merge(_) => Err(Error::IncorrectStateVariant), BlindedPayloadRef::Capella(_) => Err(Error::IncorrectStateVariant), BlindedPayloadRef::Eip4844(_) => Err(Error::IncorrectStateVariant), BlindedPayloadRef::Eip6110(inner) => { - Ok(inner.execution_payload_header.deposit_receipts_root) + Ok(inner.execution_payload_header.deposit_receipts.clone()) } } } @@ -781,7 +777,7 @@ macro_rules! impl_exec_payload_common { g(self) } - fn deposit_receipts_root(&self) -> Result { + fn deposit_receipts(&self) -> Result, Error> { let h = $h; h(self) } @@ -824,11 +820,12 @@ macro_rules! impl_exec_payload_for_fork { c }, { - let c: for<'a> fn(&'a $wrapper_type_header) -> Result = - |payload: &$wrapper_type_header| { - let wrapper_ref_type = BlindedPayloadRef::$fork_variant(&payload); - wrapper_ref_type.deposit_receipts_root() - }; + let c: for<'a> fn( + &'a $wrapper_type_header, + ) -> Result, Error> = |payload: &$wrapper_type_header| { + let wrapper_ref_type = BlindedPayloadRef::$fork_variant(&payload); + wrapper_ref_type.deposit_receipts() + }; c } ); @@ -911,10 +908,10 @@ macro_rules! impl_exec_payload_for_fork { c }, { - let c: for<'a> fn(&'a $wrapper_type_full) -> Result = + let c: for<'a> fn(&'a $wrapper_type_full) -> Result, Error> = |payload: &$wrapper_type_full| { let wrapper_ref_type = FullPayloadRef::$fork_variant(&payload); - wrapper_ref_type.deposit_receipts_root() + wrapper_ref_type.deposit_receipts() }; c } From a22e4bf63681887c7a41f6bc16d3ccb7346615be Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Mon, 8 May 2023 14:58:23 -0500 Subject: [PATCH 91/96] Implement KZG EF Tests (#4274) --- Cargo.lock | 4 + beacon_node/beacon_chain/src/kzg_utils.rs | 24 +++- crypto/kzg/src/lib.rs | 27 +++- testing/ef_tests/Cargo.toml | 4 + testing/ef_tests/src/cases.rs | 12 ++ .../src/cases/kzg_blob_to_kzg_commitment.rs | 46 +++++++ .../src/cases/kzg_compute_blob_kzg_proof.rs | 51 ++++++++ .../src/cases/kzg_compute_kzg_proof.rs | 61 +++++++++ .../src/cases/kzg_verify_blob_kzg_proof.rs | 92 ++++++++++++++ .../cases/kzg_verify_blob_kzg_proof_batch.rs | 62 +++++++++ .../src/cases/kzg_verify_kzg_proof.rs | 52 ++++++++ testing/ef_tests/src/handler.rs | 120 ++++++++++++++++++ testing/ef_tests/tests/tests.rs | 30 +++++ 13 files changed, 583 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89780448df3..9a9bebab814 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2004,6 +2004,8 @@ dependencies = [ "compare_fields", "compare_fields_derive", "derivative", + "eth2_network_config", + "eth2_serde_utils", "eth2_ssz", "eth2_ssz_derive", "ethereum-types 0.14.1", @@ -2011,9 +2013,11 @@ dependencies = [ "fork_choice", "fs2", "hex", + "kzg", "rayon", "serde", "serde_derive", + "serde_json", "serde_repr", "serde_yaml", "snap", diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 19788a1b459..7091b06fb8a 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -1,5 +1,5 @@ use kzg::{Error as KzgError, Kzg, BYTES_PER_BLOB}; -use types::{Blob, EthSpec, KzgCommitment, KzgProof}; +use types::{Blob, EthSpec, Hash256, KzgCommitment, KzgProof}; /// Converts a blob ssz List object to an array to be used with the kzg /// crypto library. @@ -56,3 +56,25 @@ pub fn blob_to_kzg_commitment( ) -> Result { kzg.blob_to_kzg_commitment(ssz_blob_to_crypto_blob::(blob)) } + +/// Compute the kzg proof for a given blob and an evaluation point z. +pub fn compute_kzg_proof( + kzg: &Kzg, + blob: Blob, + z: Hash256, +) -> Result<(KzgProof, Hash256), KzgError> { + let z = z.0.into(); + kzg.compute_kzg_proof(ssz_blob_to_crypto_blob::(blob), z) + .map(|(proof, z)| (proof, Hash256::from_slice(&z.to_vec()))) +} + +/// Verify a `kzg_proof` for a `kzg_commitment` that evaluating a polynomial at `z` results in `y` +pub fn verify_kzg_proof( + kzg: &Kzg, + kzg_commitment: KzgCommitment, + kzg_proof: KzgProof, + z: Hash256, + y: Hash256, +) -> Result { + kzg.verify_kzg_proof(kzg_commitment, z.0.into(), y.0.into(), kzg_proof) +} diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index b300e2d3bb2..d0ed6728df5 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -3,11 +3,11 @@ mod kzg_proof; mod trusted_setup; pub use crate::{kzg_commitment::KzgCommitment, kzg_proof::KzgProof, trusted_setup::TrustedSetup}; -use c_kzg::Bytes48; pub use c_kzg::{ Blob, Error as CKzgError, KzgSettings, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT, FIELD_ELEMENTS_PER_BLOB, }; +use c_kzg::{Bytes32, Bytes48}; use std::path::PathBuf; #[derive(Debug)] @@ -116,4 +116,29 @@ impl Kzg { .map_err(Error::InvalidBlob) .map(|com| KzgCommitment(com.to_bytes().into_inner())) } + + /// Computes the kzg proof for a given `blob` and an evaluation point `z` + pub fn compute_kzg_proof(&self, blob: Blob, z: Bytes32) -> Result<(KzgProof, Bytes32), Error> { + c_kzg::KzgProof::compute_kzg_proof(blob, z, &self.trusted_setup) + .map_err(Error::KzgProofComputationFailed) + .map(|(proof, y)| (KzgProof(proof.to_bytes().into_inner()), y)) + } + + /// Verifies a `kzg_proof` for a `kzg_commitment` that evaluating a polynomial at `z` results in `y` + pub fn verify_kzg_proof( + &self, + kzg_commitment: KzgCommitment, + z: Bytes32, + y: Bytes32, + kzg_proof: KzgProof, + ) -> Result { + c_kzg::KzgProof::verify_kzg_proof( + kzg_commitment.into(), + z, + y, + kzg_proof.into(), + &self.trusted_setup, + ) + .map_err(Error::InvalidKzgProof) + } } diff --git a/testing/ef_tests/Cargo.toml b/testing/ef_tests/Cargo.toml index 79664a26228..be34446d9b8 100644 --- a/testing/ef_tests/Cargo.toml +++ b/testing/ef_tests/Cargo.toml @@ -17,11 +17,15 @@ compare_fields_derive = { path = "../../common/compare_fields_derive" } derivative = "2.1.1" ethereum-types = "0.14.1" hex = "0.4.2" +kzg = { path = "../../crypto/kzg" } rayon = "1.4.1" serde = "1.0.116" serde_derive = "1.0.116" +serde_json = "1.0.58" serde_repr = "0.1.6" serde_yaml = "0.8.13" +eth2_network_config = { path = "../../common/eth2_network_config" } +eth2_serde_utils = { path = "../../consensus/serde_utils" } eth2_ssz = "0.4.1" eth2_ssz_derive = "0.3.1" tree_hash = "0.4.1" diff --git a/testing/ef_tests/src/cases.rs b/testing/ef_tests/src/cases.rs index 216912a4f14..f328fa64047 100644 --- a/testing/ef_tests/src/cases.rs +++ b/testing/ef_tests/src/cases.rs @@ -18,6 +18,12 @@ mod fork; mod fork_choice; mod genesis_initialization; mod genesis_validity; +mod kzg_blob_to_kzg_commitment; +mod kzg_compute_blob_kzg_proof; +mod kzg_compute_kzg_proof; +mod kzg_verify_blob_kzg_proof; +mod kzg_verify_blob_kzg_proof_batch; +mod kzg_verify_kzg_proof; mod merkle_proof_validity; mod operations; mod rewards; @@ -42,6 +48,12 @@ pub use epoch_processing::*; pub use fork::ForkTest; pub use genesis_initialization::*; pub use genesis_validity::*; +pub use kzg_blob_to_kzg_commitment::*; +pub use kzg_compute_blob_kzg_proof::*; +pub use kzg_compute_kzg_proof::*; +pub use kzg_verify_blob_kzg_proof::*; +pub use kzg_verify_blob_kzg_proof_batch::*; +pub use kzg_verify_kzg_proof::*; pub use merkle_proof_validity::*; pub use operations::*; pub use rewards::RewardsTest; diff --git a/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs b/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs index e69de29bb2d..f160b8b2397 100644 --- a/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs +++ b/testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs @@ -0,0 +1,46 @@ +use super::*; +use crate::case_result::compare_result; +use beacon_chain::kzg_utils::blob_to_kzg_commitment; +use kzg::KzgCommitment; +use serde_derive::Deserialize; +use std::marker::PhantomData; + +#[derive(Debug, Clone, Deserialize)] +pub struct KZGBlobToKZGCommitmentInput { + pub blob: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct KZGBlobToKZGCommitment { + pub input: KZGBlobToKZGCommitmentInput, + pub output: Option, + #[serde(skip)] + _phantom: PhantomData, +} + +impl LoadCase for KZGBlobToKZGCommitment { + fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result { + decode::yaml_decode_file(path.join("data.yaml").as_path()) + } +} + +impl Case for KZGBlobToKZGCommitment { + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name == ForkName::Deneb + } + + fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { + let kzg = get_kzg()?; + + let commitment = parse_blob::(&self.input.blob).and_then(|blob| { + blob_to_kzg_commitment::(&kzg, blob).map_err(|e| { + Error::InternalError(format!("Failed to compute kzg commitment: {:?}", e)) + }) + }); + + let expected = self.output.as_ref().and_then(|s| parse_commitment(s).ok()); + + compare_result::(&commitment, &expected) + } +} diff --git a/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs index e69de29bb2d..30187f91cef 100644 --- a/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs @@ -0,0 +1,51 @@ +use super::*; +use crate::case_result::compare_result; +use beacon_chain::kzg_utils::compute_blob_kzg_proof; +use kzg::KzgProof; +use serde_derive::Deserialize; +use std::marker::PhantomData; + +#[derive(Debug, Clone, Deserialize)] +pub struct KZGComputeBlobKZGProofInput { + pub blob: String, + pub commitment: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct KZGComputeBlobKZGProof { + pub input: KZGComputeBlobKZGProofInput, + pub output: Option, + #[serde(skip)] + _phantom: PhantomData, +} + +impl LoadCase for KZGComputeBlobKZGProof { + fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result { + decode::yaml_decode_file(path.join("data.yaml").as_path()) + } +} + +impl Case for KZGComputeBlobKZGProof { + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name == ForkName::Deneb + } + + fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { + let parse_input = |input: &KZGComputeBlobKZGProofInput| -> Result<_, Error> { + let blob = parse_blob::(&input.blob)?; + let commitment = parse_commitment(&input.commitment)?; + Ok((blob, commitment)) + }; + + let kzg = get_kzg()?; + let proof = parse_input(&self.input).and_then(|(blob, commitment)| { + compute_blob_kzg_proof::(&kzg, &blob, commitment) + .map_err(|e| Error::InternalError(format!("Failed to compute kzg proof: {:?}", e))) + }); + + let expected = self.output.as_ref().and_then(|s| parse_proof(s).ok()); + + compare_result::(&proof, &expected) + } +} diff --git a/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs index e69de29bb2d..f851947d9f3 100644 --- a/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs @@ -0,0 +1,61 @@ +use super::*; +use crate::case_result::compare_result; +use beacon_chain::kzg_utils::compute_kzg_proof; +use kzg::KzgProof; +use serde_derive::Deserialize; +use std::marker::PhantomData; +use std::str::FromStr; +use types::Hash256; + +pub fn parse_point(point: &str) -> Result { + Hash256::from_str(&point[2..]) + .map_err(|e| Error::FailedToParseTest(format!("Failed to parse point: {:?}", e))) +} + +#[derive(Debug, Clone, Deserialize)] +pub struct KZGComputeKZGProofInput { + pub blob: String, + pub z: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct KZGComputeKZGProof { + pub input: KZGComputeKZGProofInput, + pub output: Option<(String, Hash256)>, + #[serde(skip)] + _phantom: PhantomData, +} + +impl LoadCase for KZGComputeKZGProof { + fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result { + decode::yaml_decode_file(path.join("data.yaml").as_path()) + } +} + +impl Case for KZGComputeKZGProof { + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name == ForkName::Deneb + } + + fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { + let parse_input = |input: &KZGComputeKZGProofInput| -> Result<_, Error> { + let blob = parse_blob::(&input.blob)?; + let z = parse_point(&input.z)?; + Ok((blob, z)) + }; + + let kzg = get_kzg()?; + let proof = parse_input(&self.input).and_then(|(blob, z)| { + compute_kzg_proof::(&kzg, blob, z) + .map_err(|e| Error::InternalError(format!("Failed to compute kzg proof: {:?}", e))) + }); + + let expected = self + .output + .as_ref() + .and_then(|(s, z)| parse_proof(s).ok().map(|proof| (proof, *z))); + + compare_result::<(KzgProof, Hash256), _>(&proof, &expected) + } +} diff --git a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs index e69de29bb2d..fdc68a59201 100644 --- a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs @@ -0,0 +1,92 @@ +use super::*; +use crate::case_result::compare_result; +use beacon_chain::kzg_utils::validate_blob; +use eth2_network_config::TRUSTED_SETUP; +use kzg::{Kzg, KzgCommitment, KzgProof, TrustedSetup}; +use serde_derive::Deserialize; +use std::convert::TryInto; +use std::marker::PhantomData; +use types::Blob; + +pub fn get_kzg() -> Result { + let trusted_setup: TrustedSetup = serde_json::from_reader(TRUSTED_SETUP) + .map_err(|e| Error::InternalError(format!("Failed to initialize kzg: {:?}", e)))?; + Kzg::new_from_trusted_setup(trusted_setup) + .map_err(|e| Error::InternalError(format!("Failed to initialize kzg: {:?}", e))) +} + +pub fn parse_proof(proof: &str) -> Result { + hex::decode(&proof[2..]) + .map_err(|e| Error::FailedToParseTest(format!("Failed to parse proof: {:?}", e))) + .and_then(|bytes| { + bytes + .try_into() + .map_err(|e| Error::FailedToParseTest(format!("Failed to parse proof: {:?}", e))) + }) + .map(KzgProof) +} + +pub fn parse_commitment(commitment: &str) -> Result { + hex::decode(&commitment[2..]) + .map_err(|e| Error::FailedToParseTest(format!("Failed to parse commitment: {:?}", e))) + .and_then(|bytes| { + bytes.try_into().map_err(|e| { + Error::FailedToParseTest(format!("Failed to parse commitment: {:?}", e)) + }) + }) + .map(KzgCommitment) +} + +pub fn parse_blob(blob: &str) -> Result, Error> { + hex::decode(&blob[2..]) + .map_err(|e| Error::FailedToParseTest(format!("Failed to parse blob: {:?}", e))) + .and_then(|bytes| { + Blob::::new(bytes) + .map_err(|e| Error::FailedToParseTest(format!("Failed to parse blob: {:?}", e))) + }) +} + +#[derive(Debug, Clone, Deserialize)] +pub struct KZGVerifyBlobKZGProofInput { + pub blob: String, + pub commitment: String, + pub proof: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct KZGVerifyBlobKZGProof { + pub input: KZGVerifyBlobKZGProofInput, + pub output: Option, + #[serde(skip)] + _phantom: PhantomData, +} + +impl LoadCase for KZGVerifyBlobKZGProof { + fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result { + decode::yaml_decode_file(path.join("data.yaml").as_path()) + } +} + +impl Case for KZGVerifyBlobKZGProof { + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name == ForkName::Deneb + } + + fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { + let parse_input = |input: &KZGVerifyBlobKZGProofInput| -> Result<(Blob, KzgCommitment, KzgProof), Error> { + let blob = parse_blob::(&input.blob)?; + let commitment = parse_commitment(&input.commitment)?; + let proof = parse_proof(&input.proof)?; + Ok((blob, commitment, proof)) + }; + + let kzg = get_kzg()?; + let result = parse_input(&self.input).and_then(|(blob, commitment, proof)| { + validate_blob::(&kzg, blob, commitment, proof) + .map_err(|e| Error::InternalError(format!("Failed to validate blob: {:?}", e))) + }); + + compare_result::(&result, &self.output) + } +} diff --git a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs index e69de29bb2d..960ad4e4f2d 100644 --- a/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs +++ b/testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs @@ -0,0 +1,62 @@ +use super::*; +use crate::case_result::compare_result; +use beacon_chain::kzg_utils::validate_blobs; +use serde_derive::Deserialize; +use std::marker::PhantomData; + +#[derive(Debug, Clone, Deserialize)] +pub struct KZGVerifyBlobKZGProofBatchInput { + pub blobs: Vec, + pub commitments: Vec, + pub proofs: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct KZGVerifyBlobKZGProofBatch { + pub input: KZGVerifyBlobKZGProofBatchInput, + pub output: Option, + #[serde(skip)] + _phantom: PhantomData, +} + +impl LoadCase for KZGVerifyBlobKZGProofBatch { + fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result { + decode::yaml_decode_file(path.join("data.yaml").as_path()) + } +} + +impl Case for KZGVerifyBlobKZGProofBatch { + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name == ForkName::Deneb + } + + fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { + let parse_input = |input: &KZGVerifyBlobKZGProofBatchInput| -> Result<_, Error> { + let blobs = input + .blobs + .iter() + .map(|s| parse_blob::(s)) + .collect::, _>>()?; + let commitments = input + .commitments + .iter() + .map(|s| parse_commitment(s)) + .collect::, _>>()?; + let proofs = input + .proofs + .iter() + .map(|s| parse_proof(s)) + .collect::, _>>()?; + Ok((commitments, blobs, proofs)) + }; + + let kzg = get_kzg()?; + let result = parse_input(&self.input).and_then(|(commitments, blobs, proofs)| { + validate_blobs::(&kzg, &commitments, &blobs, &proofs) + .map_err(|e| Error::InternalError(format!("Failed to validate blobs: {:?}", e))) + }); + + compare_result::(&result, &self.output) + } +} diff --git a/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs b/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs index e69de29bb2d..638c3b28359 100644 --- a/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs +++ b/testing/ef_tests/src/cases/kzg_verify_kzg_proof.rs @@ -0,0 +1,52 @@ +use super::*; +use crate::case_result::compare_result; +use beacon_chain::kzg_utils::verify_kzg_proof; +use serde_derive::Deserialize; +use std::marker::PhantomData; + +#[derive(Debug, Clone, Deserialize)] +pub struct KZGVerifyKZGProofInput { + pub commitment: String, + pub z: String, + pub y: String, + pub proof: String, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct KZGVerifyKZGProof { + pub input: KZGVerifyKZGProofInput, + pub output: Option, + #[serde(skip)] + _phantom: PhantomData, +} + +impl LoadCase for KZGVerifyKZGProof { + fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result { + decode::yaml_decode_file(path.join("data.yaml").as_path()) + } +} + +impl Case for KZGVerifyKZGProof { + fn is_enabled_for_fork(fork_name: ForkName) -> bool { + fork_name == ForkName::Deneb + } + + fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { + let parse_input = |input: &KZGVerifyKZGProofInput| -> Result<_, Error> { + let commitment = parse_commitment(&input.commitment)?; + let z = parse_point(&input.z)?; + let y = parse_point(&input.y)?; + let proof = parse_proof(&input.proof)?; + Ok((commitment, z, y, proof)) + }; + + let kzg = get_kzg()?; + let result = parse_input(&self.input).and_then(|(commitment, z, y, proof)| { + verify_kzg_proof::(&kzg, commitment, proof, z, y) + .map_err(|e| Error::InternalError(format!("Failed to validate proof: {:?}", e))) + }); + + compare_result::(&result, &self.output) + } +} diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index e6ca6aeaa04..6dec9346291 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -637,6 +637,126 @@ impl Handler for GenesisInitializationHandler { } } +#[derive(Derivative)] +#[derivative(Default(bound = ""))] +pub struct KZGBlobToKZGCommitmentHandler(PhantomData); + +impl Handler for KZGBlobToKZGCommitmentHandler { + type Case = cases::KZGBlobToKZGCommitment; + + fn config_name() -> &'static str { + "general" + } + + fn runner_name() -> &'static str { + "kzg" + } + + fn handler_name(&self) -> String { + "blob_to_kzg_commitment".into() + } +} + +#[derive(Derivative)] +#[derivative(Default(bound = ""))] +pub struct KZGComputeBlobKZGProofHandler(PhantomData); + +impl Handler for KZGComputeBlobKZGProofHandler { + type Case = cases::KZGComputeBlobKZGProof; + + fn config_name() -> &'static str { + "general" + } + + fn runner_name() -> &'static str { + "kzg" + } + + fn handler_name(&self) -> String { + "compute_blob_kzg_proof".into() + } +} + +#[derive(Derivative)] +#[derivative(Default(bound = ""))] +pub struct KZGComputeKZGProofHandler(PhantomData); + +impl Handler for KZGComputeKZGProofHandler { + type Case = cases::KZGComputeKZGProof; + + fn config_name() -> &'static str { + "general" + } + + fn runner_name() -> &'static str { + "kzg" + } + + fn handler_name(&self) -> String { + "compute_kzg_proof".into() + } +} + +#[derive(Derivative)] +#[derivative(Default(bound = ""))] +pub struct KZGVerifyBlobKZGProofHandler(PhantomData); + +impl Handler for KZGVerifyBlobKZGProofHandler { + type Case = cases::KZGVerifyBlobKZGProof; + + fn config_name() -> &'static str { + "general" + } + + fn runner_name() -> &'static str { + "kzg" + } + + fn handler_name(&self) -> String { + "verify_blob_kzg_proof".into() + } +} + +#[derive(Derivative)] +#[derivative(Default(bound = ""))] +pub struct KZGVerifyBlobKZGProofBatchHandler(PhantomData); + +impl Handler for KZGVerifyBlobKZGProofBatchHandler { + type Case = cases::KZGVerifyBlobKZGProofBatch; + + fn config_name() -> &'static str { + "general" + } + + fn runner_name() -> &'static str { + "kzg" + } + + fn handler_name(&self) -> String { + "verify_blob_kzg_proof_batch".into() + } +} + +#[derive(Derivative)] +#[derivative(Default(bound = ""))] +pub struct KZGVerifyKZGProofHandler(PhantomData); + +impl Handler for KZGVerifyKZGProofHandler { + type Case = cases::KZGVerifyKZGProof; + + fn config_name() -> &'static str { + "general" + } + + fn runner_name() -> &'static str { + "kzg" + } + + fn handler_name(&self) -> String { + "verify_kzg_proof".into() + } +} + #[derive(Derivative)] #[derivative(Default(bound = ""))] pub struct MerkleProofValidityHandler(PhantomData); diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 2bd8580bf2a..7d9d8d90853 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -563,6 +563,36 @@ fn genesis_validity() { // Note: there are no genesis validity tests for mainnet } +#[test] +fn kzg_blob_to_kzg_commitment() { + KZGBlobToKZGCommitmentHandler::::default().run(); +} + +#[test] +fn kzg_compute_blob_kzg_proof() { + KZGComputeBlobKZGProofHandler::::default().run(); +} + +#[test] +fn kzg_compute_kzg_proof() { + KZGComputeKZGProofHandler::::default().run(); +} + +#[test] +fn kzg_verify_blob_kzg_proof() { + KZGVerifyBlobKZGProofHandler::::default().run(); +} + +#[test] +fn kzg_verify_blob_kzg_proof_batch() { + KZGVerifyBlobKZGProofBatchHandler::::default().run(); +} + +#[test] +fn kzg_verify_kzg_proof() { + KZGVerifyKZGProofHandler::::default().run(); +} + #[test] fn merkle_proof_validity() { MerkleProofValidityHandler::::default().run(); From 46db30416d8b85f0c8c10a26bc6bc1b39b60ccca Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Fri, 12 May 2023 09:08:24 -0500 Subject: [PATCH 92/96] Implement Overflow LRU Cache for Pending Blobs (#4203) * All Necessary Objects Implement Encode/Decode * Major Components for LRUOverflowCache Implemented * Finish Database Code * Add Maintenance Methods * Added Maintenance Service * Persist Blobs on Shutdown / Reload on Startup * Address Clippy Complaints * Add (emum_behaviour = "tag") to ssz_derive * Convert Encode/Decode Implementations to "tag" * Started Adding Tests * Added a ton of tests * 1 character fix * Feature Guard Minimal Spec Tests * Update beacon_node/beacon_chain/src/data_availability_checker.rs Co-authored-by: realbigsean * Address Sean's Comments * Add iter_raw_keys method * Remove TODOs --------- Co-authored-by: realbigsean --- beacon_node/beacon_chain/src/beacon_chain.rs | 10 +- .../beacon_chain/src/blob_verification.rs | 4 +- .../beacon_chain/src/block_verification.rs | 11 +- beacon_node/beacon_chain/src/builder.rs | 6 +- .../src/data_availability_checker.rs | 450 ++--- .../overflow_lru_cache.rs | 1645 +++++++++++++++++ beacon_node/beacon_chain/src/errors.rs | 3 + .../src/eth1_finalization_cache.rs | 3 +- beacon_node/beacon_chain/src/metrics.rs | 2 + beacon_node/client/src/builder.rs | 5 + .../test_utils/execution_block_generator.rs | 3 +- .../beacon_processor/worker/rpc_methods.rs | 2 +- beacon_node/store/src/leveldb_store.rs | 30 + beacon_node/store/src/lib.rs | 13 + consensus/fork_choice/src/fork_choice.rs | 3 +- consensus/ssz_derive/src/lib.rs | 145 +- consensus/ssz_derive/tests/tests.rs | 15 + .../state_processing/src/consensus_context.rs | 5 +- consensus/types/src/beacon_block.rs | 21 +- consensus/types/src/beacon_state.rs | 98 +- consensus/types/src/fork_name.rs | 4 +- consensus/types/src/lib.rs | 6 +- consensus/types/src/signed_beacon_block.rs | 133 ++ 23 files changed, 2364 insertions(+), 253 deletions(-) create mode 100644 beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index e7180cae141..ea4e3179b26 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -465,7 +465,7 @@ pub struct BeaconChain { /// Provides monitoring of a set of explicitly defined validators. pub validator_monitor: RwLock>, pub proposal_blob_cache: BlobCache, - pub data_availability_checker: DataAvailabilityChecker, + pub data_availability_checker: DataAvailabilityChecker, pub kzg: Option>, } @@ -609,6 +609,13 @@ impl BeaconChain { Ok(()) } + pub fn persist_data_availabilty_checker(&self) -> Result<(), Error> { + let _timer = metrics::start_timer(&metrics::PERSIST_DATA_AVAILABILITY_CHECKER); + self.data_availability_checker.persist_all()?; + + Ok(()) + } + /// Returns the slot _right now_ according to `self.slot_clock`. Returns `Err` if the slot is /// unavailable. /// @@ -6268,6 +6275,7 @@ impl Drop for BeaconChain { let drop = || -> Result<(), Error> { self.persist_head_and_fork_choice()?; self.persist_op_pool()?; + self.persist_data_availabilty_checker()?; self.persist_eth1_cache() }; diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index d5e5e9665a5..2216764c278 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -15,6 +15,7 @@ use crate::BeaconChainError; use eth2::types::BlockContentsTuple; use kzg::Kzg; use slog::{debug, warn}; +use ssz_derive::{Decode, Encode}; use std::borrow::Cow; use types::{ BeaconBlockRef, BeaconState, BeaconStateError, BlobSidecar, BlobSidecarList, ChainSpec, @@ -398,8 +399,9 @@ fn cheap_state_advance_to_obtain_committees<'a, E: EthSpec>( /// Wrapper over a `BlobSidecar` for which we have completed kzg verification. /// i.e. `verify_blob_kzg_proof(blob, commitment, proof) == true`. -#[derive(Debug, Derivative, Clone)] +#[derive(Debug, Derivative, Clone, Encode, Decode)] #[derivative(PartialEq, Eq)] +#[ssz(struct_behaviour = "transparent")] pub struct KzgVerifiedBlob { blob: Arc>, } diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 94316c0d309..2107fbf6935 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -77,6 +77,7 @@ use safe_arith::ArithError; use slog::{debug, error, warn, Logger}; use slot_clock::SlotClock; use ssz::Encode; +use ssz_derive::{Decode, Encode}; use state_processing::per_block_processing::{errors::IntoWithIndex, is_merge_transition_block}; use state_processing::{ block_signature_verifier::{BlockSignatureVerifier, Error as BlockSignatureVerifierError}, @@ -95,6 +96,7 @@ use task_executor::JoinHandle; use tree_hash::TreeHash; use types::blob_sidecar::BlobIdentifier; use types::ExecPayload; +use types::{ssz_tagged_beacon_state, ssz_tagged_signed_beacon_block}; use types::{ BeaconBlockRef, BeaconState, BeaconStateError, BlindedPayload, ChainSpec, CloneConfig, Epoch, EthSpec, ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, @@ -499,7 +501,7 @@ impl From for BlockError { } /// Stores information about verifying a payload against an execution engine. -#[derive(Clone)] +#[derive(Debug, PartialEq, Clone, Encode, Decode)] pub struct PayloadVerificationOutcome { pub payload_verification_status: PayloadVerificationStatus, pub is_valid_merge_transition_block: bool, @@ -718,6 +720,7 @@ impl ExecutedBlock { } } +#[derive(Debug, PartialEq)] pub struct AvailableExecutedBlock { pub block: AvailableBlock, pub import_data: BlockImportData, @@ -755,6 +758,7 @@ impl AvailableExecutedBlock { } } +#[derive(Encode, Decode, Clone)] pub struct AvailabilityPendingExecutedBlock { pub block: AvailabilityPendingBlock, pub import_data: BlockImportData, @@ -799,9 +803,14 @@ impl AvailabilityPendingExecutedBlock { } } +#[derive(Debug, PartialEq, Encode, Decode, Clone)] +// TODO (mark): investigate using an Arc / Arc +// here to make this cheaper to clone pub struct BlockImportData { pub block_root: Hash256, + #[ssz(with = "ssz_tagged_beacon_state")] pub state: BeaconState, + #[ssz(with = "ssz_tagged_signed_beacon_block")] pub parent_block: SignedBeaconBlock>, pub parent_eth1_finalization_data: Eth1FinalizationData, pub confirmed_state_roots: Vec, diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index a2347ede948..78f39e35810 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -794,7 +794,7 @@ where let beacon_chain = BeaconChain { spec: self.spec.clone(), config: self.chain_config, - store, + store: store.clone(), task_executor: self .task_executor .ok_or("Cannot build without task executor")?, @@ -864,8 +864,10 @@ where data_availability_checker: DataAvailabilityChecker::new( slot_clock, kzg.clone(), + store, self.spec, - ), + ) + .map_err(|e| format!("Error initializing DataAvailabiltyChecker: {:?}", e))?, proposal_blob_cache: BlobCache::default(), kzg, }; diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 1b44947c08a..550515009ec 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -4,23 +4,29 @@ use crate::blob_verification::{ }; use crate::block_verification::{AvailabilityPendingExecutedBlock, AvailableExecutedBlock}; +use crate::data_availability_checker::overflow_lru_cache::OverflowLRUCache; +use crate::{BeaconChain, BeaconChainTypes, BeaconStore}; use kzg::Error as KzgError; use kzg::Kzg; -use parking_lot::RwLock; +use slog::{debug, error}; use slot_clock::SlotClock; -use ssz_types::{Error, FixedVector, VariableList}; +use ssz_types::{Error, VariableList}; use state_processing::per_block_processing::deneb::deneb::verify_kzg_commitments_against_transactions; -use std::collections::hash_map::{Entry, OccupiedEntry}; -use std::collections::HashMap; use std::sync::Arc; +use task_executor::TaskExecutor; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::{BlobIdentifier, BlobSidecar}; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; +use types::ssz_tagged_signed_beacon_block; use types::{ BeaconBlockRef, BlobSidecarList, ChainSpec, Epoch, EthSpec, ExecPayload, FullPayload, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, }; +mod overflow_lru_cache; + +pub const OVERFLOW_LRU_CAPACITY: usize = 1024; + #[derive(Debug)] pub enum AvailabilityCheckError { DuplicateBlob(Hash256), @@ -39,6 +45,9 @@ pub enum AvailabilityCheckError { }, Pending, IncorrectFork, + BlobIndexInvalid(u64), + StoreError(store::Error), + DecodeError(ssz::DecodeError), } impl From for AvailabilityCheckError { @@ -47,70 +56,35 @@ impl From for AvailabilityCheckError { } } +impl From for AvailabilityCheckError { + fn from(value: store::Error) -> Self { + Self::StoreError(value) + } +} + +impl From for AvailabilityCheckError { + fn from(value: ssz::DecodeError) -> Self { + Self::DecodeError(value) + } +} + /// This cache contains /// - blobs that have been gossip verified /// - commitments for blocks that have been gossip verified, but the commitments themselves /// have not been verified against blobs /// - blocks that have been fully verified and only require a data availability check -pub struct DataAvailabilityChecker { - availability_cache: RwLock>>, - slot_clock: S, +pub struct DataAvailabilityChecker { + availability_cache: Arc>, + slot_clock: T::SlotClock, kzg: Option>, spec: ChainSpec, } -/// Caches partially available blobs and execution verified blocks corresponding -/// to a given `block_hash` that are received over gossip. -/// -/// The blobs are all gossip and kzg verified. -/// The block has completed all verifications except the availability check. -struct ReceivedComponents { - verified_blobs: FixedVector>, T::MaxBlobsPerBlock>, - executed_block: Option>, -} - -impl ReceivedComponents { - fn new_from_blob(blob: KzgVerifiedBlob) -> Self { - let mut verified_blobs = FixedVector::<_, _>::default(); - // TODO: verify that we've already ensured the blob index < T::MaxBlobsPerBlock - if let Some(mut_maybe_blob) = verified_blobs.get_mut(blob.blob_index() as usize) { - *mut_maybe_blob = Some(blob); - } - - Self { - verified_blobs, - executed_block: None, - } - } - - fn new_from_block(block: AvailabilityPendingExecutedBlock) -> Self { - Self { - verified_blobs: <_>::default(), - executed_block: Some(block), - } - } - - /// Returns `true` if the cache has all blobs corresponding to the - /// kzg commitments in the block. - fn has_all_blobs(&self, block: &AvailabilityPendingExecutedBlock) -> bool { - for i in 0..block.num_blobs_expected() { - if self - .verified_blobs - .get(i) - .map(|maybe_blob| maybe_blob.is_none()) - .unwrap_or(true) - { - return false; - } - } - true - } -} - /// This type is returned after adding a block / blob to the `DataAvailabilityChecker`. /// /// Indicates if the block is fully `Available` or if we need blobs or blocks /// to "complete" the requirements for an `AvailableBlock`. +#[derive(Debug, PartialEq)] pub enum Availability { PendingBlobs(Vec), PendingBlock(Hash256), @@ -129,25 +103,28 @@ impl Availability { } } -impl DataAvailabilityChecker { - pub fn new(slot_clock: S, kzg: Option>, spec: ChainSpec) -> Self { - Self { - availability_cache: <_>::default(), +impl DataAvailabilityChecker { + pub fn new( + slot_clock: T::SlotClock, + kzg: Option>, + store: BeaconStore, + spec: ChainSpec, + ) -> Result { + let overflow_cache = OverflowLRUCache::new(OVERFLOW_LRU_CAPACITY, store)?; + Ok(Self { + availability_cache: Arc::new(overflow_cache), slot_clock, kzg, spec, - } + }) } /// Get a blob from the availability cache. - pub fn get_blob(&self, blob_id: &BlobIdentifier) -> Option>> { - self.availability_cache - .read() - .get(&blob_id.block_root)? - .verified_blobs - .get(blob_id.index as usize)? - .as_ref() - .map(|kzg_verified_blob| kzg_verified_blob.clone_blob()) + pub fn get_blob( + &self, + blob_id: &BlobIdentifier, + ) -> Result>>, AvailabilityCheckError> { + self.availability_cache.peek_blob(blob_id) } /// This first validates the KZG commitments included in the blob sidecar. @@ -158,10 +135,8 @@ impl DataAvailabilityChecker { /// This should only accept gossip verified blobs, so we should not have to worry about dupes. pub fn put_gossip_blob( &self, - gossip_blob: GossipVerifiedBlob, - ) -> Result, AvailabilityCheckError> { - let block_root = gossip_blob.block_root(); - + gossip_blob: GossipVerifiedBlob, + ) -> Result, AvailabilityCheckError> { // Verify the KZG commitments. let kzg_verified_blob = if let Some(kzg) = self.kzg.as_ref() { verify_kzg_for_blob(gossip_blob, kzg)? @@ -169,125 +144,26 @@ impl DataAvailabilityChecker { return Err(AvailabilityCheckError::KzgNotInitialized); }; - let availability = match self - .availability_cache - .write() - .entry(kzg_verified_blob.block_root()) - { - Entry::Occupied(mut occupied_entry) => { - // All blobs reaching this cache should be gossip verified and gossip verification - // should filter duplicates, as well as validate indices. - let received_components = occupied_entry.get_mut(); - - if let Some(maybe_verified_blob) = received_components - .verified_blobs - .get_mut(kzg_verified_blob.blob_index() as usize) - { - *maybe_verified_blob = Some(kzg_verified_blob) - } - - if let Some(executed_block) = received_components.executed_block.take() { - self.check_block_availability_maybe_cache(occupied_entry, executed_block)? - } else { - Availability::PendingBlock(block_root) - } - } - Entry::Vacant(vacant_entry) => { - let block_root = kzg_verified_blob.block_root(); - vacant_entry.insert(ReceivedComponents::new_from_blob(kzg_verified_blob)); - Availability::PendingBlock(block_root) - } - }; - - Ok(availability) + self.availability_cache + .put_kzg_verified_blob(kzg_verified_blob) } /// Check if we have all the blobs for a block. If we do, return the Availability variant that /// triggers import of the block. pub fn put_pending_executed_block( &self, - executed_block: AvailabilityPendingExecutedBlock, - ) -> Result, AvailabilityCheckError> { - let availability = match self - .availability_cache - .write() - .entry(executed_block.import_data.block_root) - { - Entry::Occupied(occupied_entry) => { - self.check_block_availability_maybe_cache(occupied_entry, executed_block)? - } - Entry::Vacant(vacant_entry) => { - let all_blob_ids = executed_block.get_all_blob_ids(); - vacant_entry.insert(ReceivedComponents::new_from_block(executed_block)); - Availability::PendingBlobs(all_blob_ids) - } - }; - - Ok(availability) - } - - /// Checks if the provided `executed_block` contains all required blobs to be considered an - /// `AvailableBlock` based on blobs that are cached. - /// - /// Returns an error if there was an error when matching the block commitments against blob commitments. - /// - /// Returns `Ok(Availability::Available(_))` if all blobs for the block are present in cache. - /// Returns `Ok(Availability::PendingBlobs(_))` if all corresponding blobs have not been received in the cache. - fn check_block_availability_maybe_cache( - &self, - mut occupied_entry: OccupiedEntry>, - executed_block: AvailabilityPendingExecutedBlock, - ) -> Result, AvailabilityCheckError> { - if occupied_entry.get().has_all_blobs(&executed_block) { - let num_blobs_expected = executed_block.num_blobs_expected(); - let AvailabilityPendingExecutedBlock { - block, - import_data, - payload_verification_outcome, - } = executed_block; - - let ReceivedComponents { - verified_blobs, - executed_block: _, - } = occupied_entry.remove(); - - let verified_blobs = Vec::from(verified_blobs) - .into_iter() - .take(num_blobs_expected) - .map(|maybe_blob| maybe_blob.ok_or(AvailabilityCheckError::MissingBlobs)) - .collect::, _>>()?; - - let available_block = self.make_available(block, verified_blobs)?; - Ok(Availability::Available(Box::new( - AvailableExecutedBlock::new( - available_block, - import_data, - payload_verification_outcome, - ), - ))) - } else { - let received_components = occupied_entry.get_mut(); - - let missing_blob_ids = executed_block.get_filtered_blob_ids(|index| { - received_components - .verified_blobs - .get(index as usize) - .map(|maybe_blob| maybe_blob.is_none()) - .unwrap_or(true) - }); - - let _ = received_components.executed_block.insert(executed_block); - - Ok(Availability::PendingBlobs(missing_blob_ids)) - } + executed_block: AvailabilityPendingExecutedBlock, + ) -> Result, AvailabilityCheckError> { + self.availability_cache + .put_pending_executed_block(executed_block) } /// Checks if a block is available, returns a `MaybeAvailableBlock` that may include the fully /// available block. pub fn check_availability( &self, - block: BlockWrapper, - ) -> Result, AvailabilityCheckError> { + block: BlockWrapper, + ) -> Result, AvailabilityCheckError> { match block { BlockWrapper::Block(block) => self.check_availability_without_blobs(block), BlockWrapper::BlockAndBlobs(block, blob_list) => { @@ -308,8 +184,8 @@ impl DataAvailabilityChecker { /// Does not access the gossip cache. pub fn try_check_availability( &self, - block: BlockWrapper, - ) -> Result, AvailabilityCheckError> { + block: BlockWrapper, + ) -> Result, AvailabilityCheckError> { match block { BlockWrapper::Block(block) => { let blob_requirements = self.get_blob_requirements(&block)?; @@ -329,13 +205,13 @@ impl DataAvailabilityChecker { /// commitments are consistent with the provided verified blob commitments. pub fn check_availability_with_blobs( &self, - block: Arc>, - blobs: KzgVerifiedBlobList, - ) -> Result, AvailabilityCheckError> { + block: Arc>, + blobs: KzgVerifiedBlobList, + ) -> Result, AvailabilityCheckError> { match self.check_availability_without_blobs(block)? { MaybeAvailableBlock::Available(block) => Ok(block), MaybeAvailableBlock::AvailabilityPending(pending_block) => { - self.make_available(pending_block, blobs) + pending_block.make_available(blobs) } } } @@ -344,8 +220,8 @@ impl DataAvailabilityChecker { /// an AvailableBlock if no blobs are required. Otherwise this will return an AvailabilityPendingBlock. pub fn check_availability_without_blobs( &self, - block: Arc>, - ) -> Result, AvailabilityCheckError> { + block: Arc>, + ) -> Result, AvailabilityCheckError> { let blob_requirements = self.get_blob_requirements(&block)?; let blobs = match blob_requirements { BlobRequirements::EmptyBlobs => VerifiedBlobs::EmptyBlobs, @@ -363,50 +239,18 @@ impl DataAvailabilityChecker { })) } - /// Verifies an AvailabilityPendingBlock against a set of KZG verified blobs. - /// This does not check whether a block *should* have blobs, these checks should must have been - /// completed when producing the `AvailabilityPendingBlock`. - pub fn make_available( - &self, - block: AvailabilityPendingBlock, - blobs: Vec>, - ) -> Result, AvailabilityCheckError> { - let block_kzg_commitments = block.kzg_commitments()?; - if blobs.len() != block_kzg_commitments.len() { - return Err(AvailabilityCheckError::NumBlobsMismatch { - num_kzg_commitments: block_kzg_commitments.len(), - num_blobs: blobs.len(), - }); - } - - for (block_commitment, blob) in block_kzg_commitments.iter().zip(blobs.iter()) { - if *block_commitment != blob.kzg_commitment() { - return Err(AvailabilityCheckError::KzgCommitmentMismatch { - blob_index: blob.as_blob().index, - }); - } - } - - let blobs = VariableList::new(blobs.into_iter().map(|blob| blob.to_blob()).collect())?; - - Ok(AvailableBlock { - block: block.block, - blobs: VerifiedBlobs::Available(blobs), - }) - } - /// Determines the blob requirements for a block. Answers the question: "Does this block require /// blobs?". fn get_blob_requirements( &self, - block: &Arc>>, + block: &Arc>>, ) -> Result { let verified_blobs = if let (Ok(block_kzg_commitments), Ok(payload)) = ( block.message().body().blob_kzg_commitments(), block.message().body().execution_payload(), ) { if let Some(transactions) = payload.transactions() { - let verified = verify_kzg_commitments_against_transactions::( + let verified = verify_kzg_commitments_against_transactions::( transactions, block_kzg_commitments, ) @@ -437,7 +281,7 @@ impl DataAvailabilityChecker { self.spec.deneb_fork_epoch.and_then(|fork_epoch| { self.slot_clock .now() - .map(|slot| slot.epoch(T::slots_per_epoch())) + .map(|slot| slot.epoch(T::EthSpec::slots_per_epoch())) .map(|current_epoch| { std::cmp::max( fork_epoch, @@ -452,6 +296,96 @@ impl DataAvailabilityChecker { self.data_availability_boundary() .map_or(false, |da_epoch| block_epoch >= da_epoch) } + + /// Persist all in memory components to disk + pub fn persist_all(&self) -> Result<(), AvailabilityCheckError> { + self.availability_cache.write_all_to_disk() + } +} + +pub fn start_availability_cache_maintenance_service( + executor: TaskExecutor, + chain: Arc>, +) { + // this cache only needs to be maintained if deneb is configured + if chain.spec.deneb_fork_epoch.is_some() { + let overflow_cache = chain.data_availability_checker.availability_cache.clone(); + executor.spawn( + async move { availability_cache_maintenance_service(chain, overflow_cache).await }, + "availability_cache_service", + ); + } else { + debug!( + chain.log, + "Deneb fork not configured, not starting availability cache maintenance service" + ); + } +} + +async fn availability_cache_maintenance_service( + chain: Arc>, + overflow_cache: Arc>, +) { + let epoch_duration = chain.slot_clock.slot_duration() * T::EthSpec::slots_per_epoch() as u32; + loop { + match chain + .slot_clock + .duration_to_next_epoch(T::EthSpec::slots_per_epoch()) + { + Some(duration) => { + // this service should run 3/4 of the way through the epoch + let additional_delay = (epoch_duration * 3) / 4; + tokio::time::sleep(duration + additional_delay).await; + + let deneb_fork_epoch = match chain.spec.deneb_fork_epoch { + Some(epoch) => epoch, + None => break, // shutdown service if deneb fork epoch not set + }; + + debug!( + chain.log, + "Availability cache maintenance service firing"; + ); + + let current_epoch = match chain + .slot_clock + .now() + .map(|slot| slot.epoch(T::EthSpec::slots_per_epoch())) + { + Some(epoch) => epoch, + None => continue, // we'll have to try again next time I suppose.. + }; + + if current_epoch < deneb_fork_epoch { + // we are not in deneb yet + continue; + } + + let finalized_epoch = chain + .canonical_head + .fork_choice_read_lock() + .finalized_checkpoint() + .epoch; + // any data belonging to an epoch before this should be pruned + let cutoff_epoch = std::cmp::max( + finalized_epoch + 1, + std::cmp::max( + current_epoch.saturating_sub(*MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS), + deneb_fork_epoch, + ), + ); + + if let Err(e) = overflow_cache.do_maintenance(cutoff_epoch) { + error!(chain.log, "Failed to maintain availability cache"; "error" => ?e); + } + } + None => { + error!(chain.log, "Failed to read slot clock"); + // If we can't read the slot clock, just wait another slot. + tokio::time::sleep(chain.slot_clock.slot_duration()).await; + } + }; + } } pub enum BlobRequirements { @@ -493,6 +427,37 @@ impl AvailabilityPendingBlock { .blob_kzg_commitments() .map_err(|_| AvailabilityCheckError::IncorrectFork) } + + /// Verifies an AvailabilityPendingBlock against a set of KZG verified blobs. + /// This does not check whether a block *should* have blobs, these checks should must have been + /// completed when producing the `AvailabilityPendingBlock`. + pub fn make_available( + self, + blobs: Vec>, + ) -> Result, AvailabilityCheckError> { + let block_kzg_commitments = self.kzg_commitments()?; + if blobs.len() != block_kzg_commitments.len() { + return Err(AvailabilityCheckError::NumBlobsMismatch { + num_kzg_commitments: block_kzg_commitments.len(), + num_blobs: blobs.len(), + }); + } + + for (block_commitment, blob) in block_kzg_commitments.iter().zip(blobs.iter()) { + if *block_commitment != blob.kzg_commitment() { + return Err(AvailabilityCheckError::KzgCommitmentMismatch { + blob_index: blob.as_blob().index, + }); + } + } + + let blobs = VariableList::new(blobs.into_iter().map(|blob| blob.to_blob()).collect())?; + + Ok(AvailableBlock { + block: self.block, + blobs: VerifiedBlobs::Available(blobs), + }) + } } #[derive(Clone, Debug, PartialEq)] @@ -576,3 +541,44 @@ impl AsBlock for AvailableBlock { } } } + +// The standard implementation of Encode for SignedBeaconBlock +// requires us to use ssz(enum_behaviour = "transparent"). This +// prevents us from implementing Decode. We need to use a +// custom Encode and Decode in this wrapper object that essentially +// encodes it as if it were ssz(enum_behaviour = "union") +impl ssz::Encode for AvailabilityPendingBlock { + fn is_ssz_fixed_len() -> bool { + ssz_tagged_signed_beacon_block::encode::is_ssz_fixed_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + ssz_tagged_signed_beacon_block::encode::ssz_append(self.block.as_ref(), buf); + } + + fn ssz_bytes_len(&self) -> usize { + ssz_tagged_signed_beacon_block::encode::ssz_bytes_len(self.block.as_ref()) + } +} + +impl ssz::Decode for AvailabilityPendingBlock { + fn is_ssz_fixed_len() -> bool { + ssz_tagged_signed_beacon_block::decode::is_ssz_fixed_len() + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + Ok(Self { + block: Arc::new(ssz_tagged_signed_beacon_block::decode::from_ssz_bytes( + bytes, + )?), + }) + } +} + +#[cfg(test)] +mod test { + #[test] + fn check_encode_decode_availability_pending_block() { + // todo.. (difficult to create default beacon blocks to test) + } +} diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs new file mode 100644 index 00000000000..4ad5f57eb6d --- /dev/null +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -0,0 +1,1645 @@ +use crate::beacon_chain::BeaconStore; +use crate::blob_verification::KzgVerifiedBlob; +use crate::block_verification::{AvailabilityPendingExecutedBlock, AvailableExecutedBlock}; +use crate::data_availability_checker::{Availability, AvailabilityCheckError}; +use crate::store::{DBColumn, KeyValueStore}; +use crate::BeaconChainTypes; +use lru::LruCache; +use parking_lot::{Mutex, RwLock, RwLockUpgradableReadGuard, RwLockWriteGuard}; +use ssz::{Decode, Encode}; +use ssz_derive::{Decode, Encode}; +use ssz_types::FixedVector; +use std::{collections::HashSet, sync::Arc}; +use types::blob_sidecar::BlobIdentifier; +use types::{BlobSidecar, Epoch, EthSpec, Hash256}; + +/// Caches partially available blobs and execution verified blocks corresponding +/// to a given `block_hash` that are received over gossip. +/// +/// The blobs are all gossip and kzg verified. +/// The block has completed all verifications except the availability check. +#[derive(Encode, Decode, Clone)] +pub struct PendingComponents { + verified_blobs: FixedVector>, T::MaxBlobsPerBlock>, + executed_block: Option>, +} + +impl PendingComponents { + pub fn new_from_blob(blob: KzgVerifiedBlob) -> Self { + let mut verified_blobs = FixedVector::<_, _>::default(); + if let Some(mut_maybe_blob) = verified_blobs.get_mut(blob.blob_index() as usize) { + *mut_maybe_blob = Some(blob); + } + + Self { + verified_blobs, + executed_block: None, + } + } + + pub fn new_from_block(block: AvailabilityPendingExecutedBlock) -> Self { + Self { + verified_blobs: <_>::default(), + executed_block: Some(block), + } + } + + /// Returns `true` if the cache has all blobs corresponding to the + /// kzg commitments in the block. + pub fn has_all_blobs(&self, block: &AvailabilityPendingExecutedBlock) -> bool { + for i in 0..block.num_blobs_expected() { + if self + .verified_blobs + .get(i) + .map(|maybe_blob| maybe_blob.is_none()) + .unwrap_or(true) + { + return false; + } + } + true + } + + pub fn empty() -> Self { + Self { + verified_blobs: <_>::default(), + executed_block: None, + } + } + + pub fn epoch(&self) -> Option { + self.executed_block + .as_ref() + .map(|pending_block| pending_block.block.as_block().epoch()) + .or_else(|| { + for maybe_blob in self.verified_blobs.iter() { + if maybe_blob.is_some() { + return maybe_blob.as_ref().map(|kzg_verified_blob| { + kzg_verified_blob.as_blob().slot.epoch(T::slots_per_epoch()) + }); + } + } + None + }) + } +} + +#[derive(Debug, PartialEq)] +enum OverflowKey { + Block(Hash256), + Blob(Hash256, u8), +} + +impl OverflowKey { + pub fn from_block_root(block_root: Hash256) -> Self { + Self::Block(block_root) + } + + pub fn from_blob_id( + blob_id: BlobIdentifier, + ) -> Result { + if blob_id.index > E::max_blobs_per_block() as u64 || blob_id.index > u8::MAX as u64 { + return Err(AvailabilityCheckError::BlobIndexInvalid(blob_id.index)); + } + Ok(Self::Blob(blob_id.block_root, blob_id.index as u8)) + } + + pub fn root(&self) -> &Hash256 { + match self { + Self::Block(root) => root, + Self::Blob(root, _) => root, + } + } +} + +/// A wrapper around BeaconStore that implements various +/// methods used for saving and retrieving blocks / blobs +/// from the store (for organization) +struct OverflowStore(BeaconStore); + +impl OverflowStore { + pub fn persist_pending_components( + &self, + block_root: Hash256, + mut pending_components: PendingComponents, + ) -> Result<(), AvailabilityCheckError> { + let col = DBColumn::OverflowLRUCache; + + if let Some(block) = pending_components.executed_block.take() { + let key = OverflowKey::from_block_root(block_root); + self.0 + .hot_db + .put_bytes(col.as_str(), &key.as_ssz_bytes(), &block.as_ssz_bytes())? + } + + for blob in Vec::from(pending_components.verified_blobs) + .into_iter() + .flatten() + { + let key = OverflowKey::from_blob_id::(BlobIdentifier { + block_root, + index: blob.blob_index(), + })?; + + self.0 + .hot_db + .put_bytes(col.as_str(), &key.as_ssz_bytes(), &blob.as_ssz_bytes())? + } + + Ok(()) + } + + pub fn get_pending_components( + &self, + block_root: Hash256, + ) -> Result>, AvailabilityCheckError> { + // read everything from disk and reconstruct + let mut maybe_pending_components = None; + for res in self + .0 + .hot_db + .iter_raw_entries(DBColumn::OverflowLRUCache, block_root.as_bytes()) + { + let (key_bytes, value_bytes) = res?; + match OverflowKey::from_ssz_bytes(&key_bytes)? { + OverflowKey::Block(_) => { + maybe_pending_components + .get_or_insert_with(PendingComponents::empty) + .executed_block = Some(AvailabilityPendingExecutedBlock::from_ssz_bytes( + value_bytes.as_slice(), + )?); + } + OverflowKey::Blob(_, index) => { + *maybe_pending_components + .get_or_insert_with(PendingComponents::empty) + .verified_blobs + .get_mut(index as usize) + .ok_or(AvailabilityCheckError::BlobIndexInvalid(index as u64))? = + Some(KzgVerifiedBlob::from_ssz_bytes(value_bytes.as_slice())?); + } + } + } + + Ok(maybe_pending_components) + } + + // returns the hashes of all the blocks we have data for on disk + pub fn read_keys_on_disk(&self) -> Result, AvailabilityCheckError> { + let mut disk_keys = HashSet::new(); + for res in self.0.hot_db.iter_raw_keys(DBColumn::OverflowLRUCache, &[]) { + let key_bytes = res?; + disk_keys.insert(*OverflowKey::from_ssz_bytes(&key_bytes)?.root()); + } + Ok(disk_keys) + } + + pub fn load_blob( + &self, + blob_id: &BlobIdentifier, + ) -> Result>>, AvailabilityCheckError> { + let key = OverflowKey::from_blob_id::(blob_id.clone())?; + + self.0 + .hot_db + .get_bytes(DBColumn::OverflowLRUCache.as_str(), &key.as_ssz_bytes())? + .map(|blob_bytes| Arc::>::from_ssz_bytes(blob_bytes.as_slice())) + .transpose() + .map_err(|e| e.into()) + } + + pub fn delete_keys(&self, keys: &Vec) -> Result<(), AvailabilityCheckError> { + for key in keys { + self.0 + .hot_db + .key_delete(DBColumn::OverflowLRUCache.as_str(), &key.as_ssz_bytes())?; + } + Ok(()) + } +} + +// This data is protected by an RwLock +struct Critical { + pub in_memory: LruCache>, + pub store_keys: HashSet, +} + +impl Critical { + pub fn new(capacity: usize) -> Self { + Self { + in_memory: LruCache::new(capacity), + store_keys: HashSet::new(), + } + } + + pub fn reload_store_keys( + &mut self, + overflow_store: &OverflowStore, + ) -> Result<(), AvailabilityCheckError> { + let disk_keys = overflow_store.read_keys_on_disk()?; + self.store_keys = disk_keys; + Ok(()) + } + + /// This only checks for the blobs in memory + pub fn peek_blob( + &self, + blob_id: &BlobIdentifier, + ) -> Result>>, AvailabilityCheckError> { + if let Some(pending_components) = self.in_memory.peek(&blob_id.block_root) { + Ok(pending_components + .verified_blobs + .get(blob_id.index as usize) + .ok_or(AvailabilityCheckError::BlobIndexInvalid(blob_id.index))? + .as_ref() + .map(|blob| blob.clone_blob())) + } else { + Ok(None) + } + } + + /// Puts the pending components in the LRU cache. If the cache + /// is at capacity, the LRU entry is written to the store first + pub fn put_pending_components( + &mut self, + block_root: Hash256, + pending_components: PendingComponents, + overflow_store: &OverflowStore, + ) -> Result<(), AvailabilityCheckError> { + if self.in_memory.len() == self.in_memory.cap() { + // cache will overflow, must write lru entry to disk + if let Some((lru_key, lru_value)) = self.in_memory.pop_lru() { + overflow_store.persist_pending_components(lru_key, lru_value)?; + self.store_keys.insert(lru_key); + } + } + self.in_memory.put(block_root, pending_components); + Ok(()) + } + + /// Removes and returns the pending_components corresponding to + /// the `block_root` or `None` if it does not exist + pub fn pop_pending_components( + &mut self, + block_root: Hash256, + store: &OverflowStore, + ) -> Result>, AvailabilityCheckError> { + match self.in_memory.pop_entry(&block_root) { + Some((_, pending_components)) => Ok(Some(pending_components)), + None => { + // not in memory, is it in the store? + if self.store_keys.remove(&block_root) { + store.get_pending_components(block_root) + } else { + Ok(None) + } + } + } + } +} + +pub struct OverflowLRUCache { + critical: RwLock>, + overflow_store: OverflowStore, + maintenance_lock: Mutex<()>, + capacity: usize, +} + +impl OverflowLRUCache { + pub fn new( + capacity: usize, + beacon_store: BeaconStore, + ) -> Result { + let overflow_store = OverflowStore(beacon_store); + let mut critical = Critical::new(capacity); + critical.reload_store_keys(&overflow_store)?; + Ok(Self { + critical: RwLock::new(critical), + overflow_store, + maintenance_lock: Mutex::new(()), + capacity, + }) + } + + pub fn peek_blob( + &self, + blob_id: &BlobIdentifier, + ) -> Result>>, AvailabilityCheckError> { + let read_lock = self.critical.read(); + if let Some(blob) = read_lock.peek_blob(blob_id)? { + Ok(Some(blob)) + } else if read_lock.store_keys.contains(&blob_id.block_root) { + drop(read_lock); + self.overflow_store.load_blob(blob_id) + } else { + Ok(None) + } + } + + pub fn put_kzg_verified_blob( + &self, + kzg_verified_blob: KzgVerifiedBlob, + ) -> Result, AvailabilityCheckError> { + let mut write_lock = self.critical.write(); + let block_root = kzg_verified_blob.block_root(); + + let availability = if let Some(mut pending_components) = + write_lock.pop_pending_components(block_root, &self.overflow_store)? + { + let blob_index = kzg_verified_blob.blob_index(); + *pending_components + .verified_blobs + .get_mut(blob_index as usize) + .ok_or(AvailabilityCheckError::BlobIndexInvalid(blob_index))? = + Some(kzg_verified_blob); + + if let Some(executed_block) = pending_components.executed_block.take() { + self.check_block_availability_maybe_cache( + write_lock, + block_root, + pending_components, + executed_block, + )? + } else { + write_lock.put_pending_components( + block_root, + pending_components, + &self.overflow_store, + )?; + Availability::PendingBlock(block_root) + } + } else { + // not in memory or store -> put new in memory + let new_pending_components = PendingComponents::new_from_blob(kzg_verified_blob); + write_lock.put_pending_components( + block_root, + new_pending_components, + &self.overflow_store, + )?; + Availability::PendingBlock(block_root) + }; + + Ok(availability) + } + + /// Check if we have all the blobs for a block. If we do, return the Availability variant that + /// triggers import of the block. + pub fn put_pending_executed_block( + &self, + executed_block: AvailabilityPendingExecutedBlock, + ) -> Result, AvailabilityCheckError> { + let mut write_lock = self.critical.write(); + let block_root = executed_block.import_data.block_root; + + let availability = + match write_lock.pop_pending_components(block_root, &self.overflow_store)? { + Some(pending_components) => self.check_block_availability_maybe_cache( + write_lock, + block_root, + pending_components, + executed_block, + )?, + None => { + let all_blob_ids = executed_block.get_all_blob_ids(); + if all_blob_ids.is_empty() { + // no blobs for this block, we can import it + let AvailabilityPendingExecutedBlock { + block, + import_data, + payload_verification_outcome, + } = executed_block; + let available_block = block.make_available(vec![])?; + return Ok(Availability::Available(Box::new( + AvailableExecutedBlock::new( + available_block, + import_data, + payload_verification_outcome, + ), + ))); + } + let new_pending_components = PendingComponents::new_from_block(executed_block); + write_lock.put_pending_components( + block_root, + new_pending_components, + &self.overflow_store, + )?; + Availability::PendingBlobs(all_blob_ids) + } + }; + + Ok(availability) + } + + /// Checks if the provided `executed_block` contains all required blobs to be considered an + /// `AvailableBlock` based on blobs that are cached. + /// + /// Returns an error if there was an error when matching the block commitments against blob commitments. + /// + /// Returns `Ok(Availability::Available(_))` if all blobs for the block are present in cache. + /// Returns `Ok(Availability::PendingBlobs(_))` if all corresponding blobs have not been received in the cache. + fn check_block_availability_maybe_cache( + &self, + mut write_lock: RwLockWriteGuard>, + block_root: Hash256, + mut pending_components: PendingComponents, + executed_block: AvailabilityPendingExecutedBlock, + ) -> Result, AvailabilityCheckError> { + if pending_components.has_all_blobs(&executed_block) { + let num_blobs_expected = executed_block.num_blobs_expected(); + let AvailabilityPendingExecutedBlock { + block, + import_data, + payload_verification_outcome, + } = executed_block; + + let verified_blobs = Vec::from(pending_components.verified_blobs) + .into_iter() + .take(num_blobs_expected) + .map(|maybe_blob| maybe_blob.ok_or(AvailabilityCheckError::MissingBlobs)) + .collect::, _>>()?; + + let available_block = block.make_available(verified_blobs)?; + Ok(Availability::Available(Box::new( + AvailableExecutedBlock::new( + available_block, + import_data, + payload_verification_outcome, + ), + ))) + } else { + let missing_blob_ids = executed_block.get_filtered_blob_ids(|index| { + pending_components + .verified_blobs + .get(index as usize) + .map(|maybe_blob| maybe_blob.is_none()) + .unwrap_or(true) + }); + + let _ = pending_components.executed_block.insert(executed_block); + write_lock.put_pending_components( + block_root, + pending_components, + &self.overflow_store, + )?; + + Ok(Availability::PendingBlobs(missing_blob_ids)) + } + } + + // writes all in_memory objects to disk + pub fn write_all_to_disk(&self) -> Result<(), AvailabilityCheckError> { + let maintenance_lock = self.maintenance_lock.lock(); + let mut critical_lock = self.critical.write(); + + let mut swap_lru = LruCache::new(self.capacity); + std::mem::swap(&mut swap_lru, &mut critical_lock.in_memory); + + for (root, pending_components) in swap_lru.into_iter() { + self.overflow_store + .persist_pending_components(root, pending_components)?; + critical_lock.store_keys.insert(root); + } + + drop(critical_lock); + drop(maintenance_lock); + Ok(()) + } + + // maintain the cache + pub fn do_maintenance(&self, cutoff_epoch: Epoch) -> Result<(), AvailabilityCheckError> { + // ensure memory usage is below threshold + let threshold = self.capacity * 3 / 4; + self.maintain_threshold(threshold, cutoff_epoch)?; + // clean up any keys on the disk that shouldn't be there + self.prune_disk(cutoff_epoch)?; + Ok(()) + } + + fn maintain_threshold( + &self, + threshold: usize, + cutoff_epoch: Epoch, + ) -> Result<(), AvailabilityCheckError> { + // ensure only one thread at a time can be deleting things from the disk or + // moving things between memory and storage + let maintenance_lock = self.maintenance_lock.lock(); + + let mut stored = self.critical.read().in_memory.len(); + while stored > threshold { + let read_lock = self.critical.upgradable_read(); + let lru_entry = read_lock + .in_memory + .peek_lru() + .map(|(key, value)| (*key, value.clone())); + + let (lru_root, lru_pending_components) = match lru_entry { + Some((r, p)) => (r, p), + None => break, + }; + + if lru_pending_components + .epoch() + .map(|epoch| epoch < cutoff_epoch) + .unwrap_or(true) + { + // this data is no longer needed -> delete it + let mut write_lock = RwLockUpgradableReadGuard::upgrade(read_lock); + write_lock.in_memory.pop_entry(&lru_root); + stored = write_lock.in_memory.len(); + continue; + } else { + drop(read_lock); + } + + // write the lru entry to disk (we aren't holding any critical locks while we do this) + self.overflow_store + .persist_pending_components(lru_root, lru_pending_components)?; + // now that we've written to disk, grab the critical write lock + let mut write_lock = self.critical.write(); + if let Some((new_lru_root_ref, _)) = write_lock.in_memory.peek_lru() { + // need to ensure the entry we just wrote to disk wasn't updated + // while we were writing and is still the LRU entry + if *new_lru_root_ref == lru_root { + // it is still LRU entry -> delete it from memory & record that it's on disk + write_lock.in_memory.pop_entry(&lru_root); + write_lock.store_keys.insert(lru_root); + stored = write_lock.in_memory.len(); + } + } + drop(write_lock); + } + + drop(maintenance_lock); + Ok(()) + } + + fn prune_disk(&self, cutoff_epoch: Epoch) -> Result<(), AvailabilityCheckError> { + // ensure only one thread at a time can be deleting things from the disk or + // moving things between memory and storage + let maintenance_lock = self.maintenance_lock.lock(); + + struct BlockData { + keys: Vec, + root: Hash256, + epoch: Epoch, + } + + let delete_if_outdated = |cache: &OverflowLRUCache, + block_data: Option| + -> Result<(), AvailabilityCheckError> { + let block_data = match block_data { + Some(block_data) => block_data, + None => return Ok(()), + }; + let not_in_store_keys = !cache.critical.read().store_keys.contains(&block_data.root); + if not_in_store_keys { + // these keys aren't supposed to be on disk + cache.overflow_store.delete_keys(&block_data.keys)?; + } else { + // check this data is still relevant + if block_data.epoch < cutoff_epoch { + // this data is no longer needed -> delete it + self.overflow_store.delete_keys(&block_data.keys)?; + } + } + Ok(()) + }; + + let mut current_block_data: Option = None; + for res in self + .overflow_store + .0 + .hot_db + .iter_raw_entries(DBColumn::OverflowLRUCache, &[]) + { + let (key_bytes, value_bytes) = res?; + let overflow_key = OverflowKey::from_ssz_bytes(&key_bytes)?; + let current_root = *overflow_key.root(); + + match &mut current_block_data { + Some(block_data) if block_data.root == current_root => { + // still dealing with the same block + block_data.keys.push(overflow_key); + } + _ => { + // first time encountering data for this block + delete_if_outdated(self, current_block_data)?; + let current_epoch = match &overflow_key { + OverflowKey::Block(_) => { + AvailabilityPendingExecutedBlock::::from_ssz_bytes( + value_bytes.as_slice(), + )? + .block + .as_block() + .epoch() + } + OverflowKey::Blob(_, _) => { + KzgVerifiedBlob::::from_ssz_bytes(value_bytes.as_slice())? + .as_blob() + .slot + .epoch(T::EthSpec::slots_per_epoch()) + } + }; + current_block_data = Some(BlockData { + keys: vec![overflow_key], + root: current_root, + epoch: current_epoch, + }); + } + } + } + // can't fall off the end + delete_if_outdated(self, current_block_data)?; + + drop(maintenance_lock); + Ok(()) + } +} + +impl ssz::Encode for OverflowKey { + fn is_ssz_fixed_len() -> bool { + true + } + + fn ssz_append(&self, buf: &mut Vec) { + match self { + OverflowKey::Block(block_hash) => { + block_hash.ssz_append(buf); + buf.push(0u8) + } + OverflowKey::Blob(block_hash, index) => { + block_hash.ssz_append(buf); + buf.push(*index + 1) + } + } + } + + fn ssz_fixed_len() -> usize { + ::ssz_fixed_len() + 1 + } + + fn ssz_bytes_len(&self) -> usize { + match self { + Self::Block(root) => root.ssz_bytes_len() + 1, + Self::Blob(root, _) => root.ssz_bytes_len() + 1, + } + } +} + +impl ssz::Decode for OverflowKey { + fn is_ssz_fixed_len() -> bool { + true + } + + fn ssz_fixed_len() -> usize { + ::ssz_fixed_len() + 1 + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let len = bytes.len(); + let h256_len = ::ssz_fixed_len(); + let expected = h256_len + 1; + + if len != expected { + Err(ssz::DecodeError::InvalidByteLength { len, expected }) + } else { + let root_bytes = bytes + .get(..h256_len) + .ok_or(ssz::DecodeError::OutOfBoundsByte { i: 0 })?; + let block_root = Hash256::from_ssz_bytes(root_bytes)?; + let id_byte = *bytes + .get(h256_len) + .ok_or(ssz::DecodeError::OutOfBoundsByte { i: h256_len })?; + match id_byte { + 0 => Ok(OverflowKey::Block(block_root)), + n => Ok(OverflowKey::Blob(block_root, n - 1)), + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + #[cfg(feature = "spec-minimal")] + use crate::{ + blob_verification::{ + validate_blob_sidecar_for_gossip, verify_kzg_for_blob, GossipVerifiedBlob, + }, + block_verification::{BlockImportData, PayloadVerificationOutcome}, + data_availability_checker::AvailabilityPendingBlock, + eth1_finalization_cache::Eth1FinalizationData, + test_utils::{BaseHarnessType, BeaconChainHarness, DiskHarnessType}, + }; + #[cfg(feature = "spec-minimal")] + use fork_choice::PayloadVerificationStatus; + #[cfg(feature = "spec-minimal")] + use logging::test_logger; + #[cfg(feature = "spec-minimal")] + use slog::{info, Logger}; + #[cfg(feature = "spec-minimal")] + use state_processing::ConsensusContext; + #[cfg(feature = "spec-minimal")] + use std::collections::{BTreeMap, HashMap, VecDeque}; + #[cfg(feature = "spec-minimal")] + use std::ops::AddAssign; + #[cfg(feature = "spec-minimal")] + use store::{HotColdDB, ItemStore, LevelDB, StoreConfig}; + #[cfg(feature = "spec-minimal")] + use tempfile::{tempdir, TempDir}; + #[cfg(feature = "spec-minimal")] + use types::beacon_state::ssz_tagged_beacon_state; + #[cfg(feature = "spec-minimal")] + use types::{ChainSpec, ExecPayload, MinimalEthSpec}; + + #[cfg(feature = "spec-minimal")] + const LOW_VALIDATOR_COUNT: usize = 32; + + #[cfg(feature = "spec-minimal")] + fn get_store_with_spec( + db_path: &TempDir, + spec: ChainSpec, + log: Logger, + ) -> Arc, LevelDB>> { + let hot_path = db_path.path().join("hot_db"); + let cold_path = db_path.path().join("cold_db"); + let config = StoreConfig::default(); + + HotColdDB::open( + &hot_path, + &cold_path, + None, + |_, _, _| Ok(()), + config, + spec, + log, + ) + .expect("disk store should initialize") + } + + // get a beacon chain harness advanced to just before deneb fork + #[cfg(feature = "spec-minimal")] + async fn get_deneb_chain( + log: Logger, + db_path: &TempDir, + ) -> BeaconChainHarness, LevelDB>> { + let altair_fork_epoch = Epoch::new(1); + let bellatrix_fork_epoch = Epoch::new(2); + let bellatrix_fork_slot = bellatrix_fork_epoch.start_slot(E::slots_per_epoch()); + let capella_fork_epoch = Epoch::new(3); + let deneb_fork_epoch = Epoch::new(4); + let deneb_fork_slot = deneb_fork_epoch.start_slot(E::slots_per_epoch()); + + let mut spec = E::default_spec(); + spec.altair_fork_epoch = Some(altair_fork_epoch); + spec.bellatrix_fork_epoch = Some(bellatrix_fork_epoch); + spec.capella_fork_epoch = Some(capella_fork_epoch); + spec.deneb_fork_epoch = Some(deneb_fork_epoch); + + let chain_store = get_store_with_spec::(db_path, spec.clone(), log.clone()); + let validators_keypairs = + types::test_utils::generate_deterministic_keypairs(LOW_VALIDATOR_COUNT); + let harness = BeaconChainHarness::builder(E::default()) + .spec(spec.clone()) + .logger(log.clone()) + .keypairs(validators_keypairs) + .fresh_disk_store(chain_store) + .mock_execution_layer() + .build(); + + // go to bellatrix slot + harness.extend_to_slot(bellatrix_fork_slot).await; + let merge_head = &harness.chain.head_snapshot().beacon_block; + assert!(merge_head.as_merge().is_ok()); + assert_eq!(merge_head.slot(), bellatrix_fork_slot); + assert!( + merge_head + .message() + .body() + .execution_payload() + .unwrap() + .is_default_with_empty_roots(), + "Merge head is default payload" + ); + // Trigger the terminal PoW block. + harness + .execution_block_generator() + .move_to_terminal_block() + .unwrap(); + // go right before deneb slot + harness.extend_to_slot(deneb_fork_slot - 1).await; + + harness + } + + #[test] + fn overflow_key_encode_decode_equality() { + type E = types::MainnetEthSpec; + let key_block = OverflowKey::Block(Hash256::random()); + let key_blob_0 = OverflowKey::from_blob_id::(BlobIdentifier { + block_root: Hash256::random(), + index: 0, + }) + .expect("should create overflow key 0"); + let key_blob_1 = OverflowKey::from_blob_id::(BlobIdentifier { + block_root: Hash256::random(), + index: 1, + }) + .expect("should create overflow key 1"); + let key_blob_2 = OverflowKey::from_blob_id::(BlobIdentifier { + block_root: Hash256::random(), + index: 2, + }) + .expect("should create overflow key 2"); + let key_blob_3 = OverflowKey::from_blob_id::(BlobIdentifier { + block_root: Hash256::random(), + index: 3, + }) + .expect("should create overflow key 3"); + + let keys = vec![key_block, key_blob_0, key_blob_1, key_blob_2, key_blob_3]; + for key in keys { + let encoded = key.as_ssz_bytes(); + let decoded = OverflowKey::from_ssz_bytes(&encoded).expect("should decode"); + assert_eq!(key, decoded, "Encoded and decoded keys should be equal"); + } + } + + #[tokio::test] + #[cfg(feature = "spec-minimal")] + async fn ssz_tagged_beacon_state_encode_decode_equality() { + type E = MinimalEthSpec; + let altair_fork_epoch = Epoch::new(1); + let altair_fork_slot = altair_fork_epoch.start_slot(E::slots_per_epoch()); + let bellatrix_fork_epoch = Epoch::new(2); + let merge_fork_slot = bellatrix_fork_epoch.start_slot(E::slots_per_epoch()); + let capella_fork_epoch = Epoch::new(3); + let capella_fork_slot = capella_fork_epoch.start_slot(E::slots_per_epoch()); + let deneb_fork_epoch = Epoch::new(4); + let deneb_fork_slot = deneb_fork_epoch.start_slot(E::slots_per_epoch()); + + let mut spec = E::default_spec(); + spec.altair_fork_epoch = Some(altair_fork_epoch); + spec.bellatrix_fork_epoch = Some(bellatrix_fork_epoch); + spec.capella_fork_epoch = Some(capella_fork_epoch); + spec.deneb_fork_epoch = Some(deneb_fork_epoch); + + let harness = BeaconChainHarness::builder(E::default()) + .spec(spec) + .logger(logging::test_logger()) + .deterministic_keypairs(LOW_VALIDATOR_COUNT) + .fresh_ephemeral_store() + .mock_execution_layer() + .build(); + + let mut state = harness.get_current_state(); + assert!(state.as_base().is_ok()); + let encoded = ssz_tagged_beacon_state::encode::as_ssz_bytes(&state); + let decoded = + ssz_tagged_beacon_state::decode::from_ssz_bytes(&encoded).expect("should decode"); + state.drop_all_caches().expect("should drop caches"); + assert_eq!(state, decoded, "Encoded and decoded states should be equal"); + + harness.extend_to_slot(altair_fork_slot).await; + + let mut state = harness.get_current_state(); + assert!(state.as_altair().is_ok()); + let encoded = ssz_tagged_beacon_state::encode::as_ssz_bytes(&state); + let decoded = + ssz_tagged_beacon_state::decode::from_ssz_bytes(&encoded).expect("should decode"); + state.drop_all_caches().expect("should drop caches"); + assert_eq!(state, decoded, "Encoded and decoded states should be equal"); + + harness.extend_to_slot(merge_fork_slot).await; + + let mut state = harness.get_current_state(); + assert!(state.as_merge().is_ok()); + let encoded = ssz_tagged_beacon_state::encode::as_ssz_bytes(&state); + let decoded = + ssz_tagged_beacon_state::decode::from_ssz_bytes(&encoded).expect("should decode"); + state.drop_all_caches().expect("should drop caches"); + assert_eq!(state, decoded, "Encoded and decoded states should be equal"); + + harness.extend_to_slot(capella_fork_slot).await; + + let mut state = harness.get_current_state(); + assert!(state.as_capella().is_ok()); + let encoded = ssz_tagged_beacon_state::encode::as_ssz_bytes(&state); + let decoded = + ssz_tagged_beacon_state::decode::from_ssz_bytes(&encoded).expect("should decode"); + state.drop_all_caches().expect("should drop caches"); + assert_eq!(state, decoded, "Encoded and decoded states should be equal"); + + harness.extend_to_slot(deneb_fork_slot).await; + + let mut state = harness.get_current_state(); + assert!(state.as_deneb().is_ok()); + let encoded = ssz_tagged_beacon_state::encode::as_ssz_bytes(&state); + let decoded = + ssz_tagged_beacon_state::decode::from_ssz_bytes(&encoded).expect("should decode"); + state.drop_all_caches().expect("should drop caches"); + assert_eq!(state, decoded, "Encoded and decoded states should be equal"); + } + + #[cfg(feature = "spec-minimal")] + async fn availability_pending_block( + harness: &BeaconChainHarness>, + log: Logger, + ) -> ( + AvailabilityPendingExecutedBlock, + Vec>, + ) + where + E: EthSpec, + Hot: ItemStore, + Cold: ItemStore, + { + let chain = &harness.chain; + let head = chain.head_snapshot(); + let parent_state = head.beacon_state.clone_with_only_committee_caches(); + + let target_slot = chain.slot().expect("should get slot") + 1; + let parent_root = head.beacon_block_root; + let parent_block = chain + .get_blinded_block(&parent_root) + .expect("should get block") + .expect("should have block"); + + let parent_eth1_finalization_data = Eth1FinalizationData { + eth1_data: parent_block.message().body().eth1_data().clone(), + eth1_deposit_index: 0, + }; + + let (signed_beacon_block_hash, (block, maybe_blobs), state) = harness + .add_block_at_slot(target_slot, parent_state) + .await + .expect("should add block"); + let block_root = signed_beacon_block_hash.into(); + assert_eq!( + block_root, + block.canonical_root(), + "block root should match" + ); + + // log kzg commitments + info!(log, "printing kzg commitments"); + for comm in Vec::from( + block + .message() + .body() + .blob_kzg_commitments() + .expect("should be deneb fork") + .clone(), + ) { + info!(log, "kzg commitment"; "commitment" => ?comm); + } + info!(log, "done printing kzg commitments"); + + let gossip_verified_blobs = if let Some(blobs) = maybe_blobs { + Vec::from(blobs) + .into_iter() + .map(|signed_blob| { + let subnet = signed_blob.message.index; + validate_blob_sidecar_for_gossip(signed_blob, subnet, &harness.chain) + .expect("should validate blob") + }) + .collect() + } else { + vec![] + }; + + let slot = block.slot(); + let apb: AvailabilityPendingBlock = AvailabilityPendingBlock { + block: Arc::new(block), + }; + + let consensus_context = ConsensusContext::::new(slot); + let import_data: BlockImportData = BlockImportData { + block_root, + state, + parent_block, + parent_eth1_finalization_data, + confirmed_state_roots: vec![], + consensus_context, + }; + + let payload_verification_outcome = PayloadVerificationOutcome { + payload_verification_status: PayloadVerificationStatus::Verified, + is_valid_merge_transition_block: false, + }; + + let availability_pending_block = AvailabilityPendingExecutedBlock { + block: apb, + import_data, + payload_verification_outcome, + }; + + (availability_pending_block, gossip_verified_blobs) + } + + #[tokio::test] + #[cfg(feature = "spec-minimal")] + async fn overflow_cache_test_insert_components() { + type E = MinimalEthSpec; + type T = DiskHarnessType; + let log = test_logger(); + let chain_db_path = tempdir().expect("should get temp dir"); + let harness: BeaconChainHarness = get_deneb_chain(log.clone(), &chain_db_path).await; + let spec = harness.spec.clone(); + let capacity = 4; + let db_path = tempdir().expect("should get temp dir"); + let test_store = get_store_with_spec::(&db_path, spec.clone(), log.clone()); + let cache = Arc::new( + OverflowLRUCache::::new(capacity, test_store).expect("should create cache"), + ); + + let (pending_block, blobs) = availability_pending_block(&harness, log.clone()).await; + let root = pending_block.import_data.block_root; + + let blobs_expected = pending_block.num_blobs_expected(); + assert_eq!( + blobs.len(), + blobs_expected, + "should have expected number of blobs" + ); + assert!( + cache.critical.read().in_memory.is_empty(), + "cache should be empty" + ); + let availability = cache + .put_pending_executed_block(pending_block) + .expect("should put block"); + if blobs_expected == 0 { + assert!( + matches!(availability, Availability::Available(_)), + "block doesn't have blobs, should be available" + ); + assert_eq!( + cache.critical.read().in_memory.len(), + 0, + "cache should be empty because we don't have blobs" + ); + } else { + assert!( + matches!(availability, Availability::PendingBlobs(_)), + "should be pending blobs" + ); + assert_eq!( + cache.critical.read().in_memory.len(), + 1, + "cache should have one block" + ); + assert!( + cache.critical.read().in_memory.peek(&root).is_some(), + "newly inserted block should exist in memory" + ); + } + + let kzg = harness + .chain + .kzg + .as_ref() + .cloned() + .expect("kzg should exist"); + for (blob_index, gossip_blob) in blobs.into_iter().enumerate() { + let kzg_verified_blob = + verify_kzg_for_blob(gossip_blob, kzg.as_ref()).expect("kzg should verify"); + let availability = cache + .put_kzg_verified_blob(kzg_verified_blob) + .expect("should put blob"); + if blob_index == blobs_expected - 1 { + assert!(matches!(availability, Availability::Available(_))); + } else { + assert!(matches!(availability, Availability::PendingBlobs(_))); + assert_eq!(cache.critical.read().in_memory.len(), 1); + } + } + assert!( + cache.critical.read().in_memory.is_empty(), + "cache should be empty now that all components available" + ); + + let (pending_block, blobs) = availability_pending_block(&harness, log.clone()).await; + let blobs_expected = pending_block.num_blobs_expected(); + assert_eq!( + blobs.len(), + blobs_expected, + "should have expected number of blobs" + ); + let root = pending_block.import_data.block_root; + for gossip_blob in blobs { + let kzg_verified_blob = + verify_kzg_for_blob(gossip_blob, kzg.as_ref()).expect("kzg should verify"); + let availability = cache + .put_kzg_verified_blob(kzg_verified_blob) + .expect("should put blob"); + assert_eq!( + availability, + Availability::PendingBlock(root), + "should be pending block" + ); + assert_eq!(cache.critical.read().in_memory.len(), 1); + } + let availability = cache + .put_pending_executed_block(pending_block) + .expect("should put block"); + assert!( + matches!(availability, Availability::Available(_)), + "block should be available: {:?}", + availability + ); + assert!( + cache.critical.read().in_memory.is_empty(), + "cache should be empty now that all components available" + ); + } + + #[tokio::test] + #[cfg(feature = "spec-minimal")] + async fn overflow_cache_test_overflow() { + type E = MinimalEthSpec; + type T = DiskHarnessType; + let log = test_logger(); + let chain_db_path = tempdir().expect("should get temp dir"); + let harness: BeaconChainHarness = get_deneb_chain(log.clone(), &chain_db_path).await; + let spec = harness.spec.clone(); + let capacity = 4; + let db_path = tempdir().expect("should get temp dir"); + let test_store = get_store_with_spec::(&db_path, spec.clone(), log.clone()); + let cache = Arc::new( + OverflowLRUCache::::new(capacity, test_store).expect("should create cache"), + ); + + let mut pending_blocks = VecDeque::new(); + let mut pending_blobs = VecDeque::new(); + let mut roots = VecDeque::new(); + while pending_blobs.len() < capacity + 1 { + let (pending_block, blobs) = availability_pending_block(&harness, log.clone()).await; + if pending_block.num_blobs_expected() == 0 { + // we need blocks with blobs + continue; + } + let root = pending_block.block.block.canonical_root(); + pending_blocks.push_back(pending_block); + pending_blobs.push_back(blobs); + roots.push_back(root); + } + + for i in 0..capacity { + cache + .put_pending_executed_block(pending_blocks.pop_front().expect("should have block")) + .expect("should put block"); + assert_eq!(cache.critical.read().in_memory.len(), i + 1); + } + for root in roots.iter().take(capacity) { + assert!(cache.critical.read().in_memory.peek(root).is_some()); + } + assert_eq!( + cache.critical.read().in_memory.len(), + capacity, + "cache should be full" + ); + // the first block should be the lru entry + assert_eq!( + *cache + .critical + .read() + .in_memory + .peek_lru() + .expect("should exist") + .0, + roots[0], + "first block should be lru" + ); + + cache + .put_pending_executed_block(pending_blocks.pop_front().expect("should have block")) + .expect("should put block"); + assert_eq!( + cache.critical.read().in_memory.len(), + capacity, + "cache should be full" + ); + assert!( + cache.critical.read().in_memory.peek(&roots[0]).is_none(), + "first block should be evicted" + ); + assert_eq!( + *cache + .critical + .read() + .in_memory + .peek_lru() + .expect("should exist") + .0, + roots[1], + "second block should be lru" + ); + + assert!(cache + .overflow_store + .get_pending_components(roots[0]) + .expect("should exist") + .is_some()); + + let threshold = capacity * 3 / 4; + cache + .maintain_threshold(threshold, Epoch::new(0)) + .expect("should maintain threshold"); + assert_eq!( + cache.critical.read().in_memory.len(), + threshold, + "cache should have been maintained" + ); + + let store_keys = cache + .overflow_store + .read_keys_on_disk() + .expect("should read keys"); + assert_eq!(store_keys.len(), 2); + assert!(store_keys.contains(&roots[0])); + assert!(store_keys.contains(&roots[1])); + assert!(cache.critical.read().store_keys.contains(&roots[0])); + assert!(cache.critical.read().store_keys.contains(&roots[1])); + + let kzg = harness + .chain + .kzg + .as_ref() + .cloned() + .expect("kzg should exist"); + + let blobs_0 = pending_blobs.pop_front().expect("should have blobs"); + let expected_blobs = blobs_0.len(); + for (blob_index, gossip_blob) in blobs_0.into_iter().enumerate() { + let kzg_verified_blob = + verify_kzg_for_blob(gossip_blob, kzg.as_ref()).expect("kzg should verify"); + let availability = cache + .put_kzg_verified_blob(kzg_verified_blob) + .expect("should put blob"); + if blob_index == expected_blobs - 1 { + assert!(matches!(availability, Availability::Available(_))); + } else { + // the first block should be brought back into memory + assert!( + cache.critical.read().in_memory.peek(&roots[0]).is_some(), + "first block should be in memory" + ); + assert!(matches!(availability, Availability::PendingBlobs(_))); + } + } + assert_eq!( + cache.critical.read().in_memory.len(), + threshold, + "cache should no longer have the first block" + ); + cache.prune_disk(Epoch::new(0)).expect("should prune disk"); + assert!( + cache + .overflow_store + .get_pending_components(roots[1]) + .expect("no error") + .is_some(), + "second block should still be on disk" + ); + assert!( + cache + .overflow_store + .get_pending_components(roots[0]) + .expect("no error") + .is_none(), + "first block should not be on disk" + ); + } + + #[tokio::test] + #[cfg(feature = "spec-minimal")] + async fn overflow_cache_test_maintenance() { + type E = MinimalEthSpec; + type T = DiskHarnessType; + let log = test_logger(); + let chain_db_path = tempdir().expect("should get temp dir"); + let harness: BeaconChainHarness = get_deneb_chain(log.clone(), &chain_db_path).await; + let spec = harness.spec.clone(); + let n_epochs = 4; + let capacity = E::slots_per_epoch() as usize; + let db_path = tempdir().expect("should get temp dir"); + let test_store = get_store_with_spec::(&db_path, spec.clone(), log.clone()); + let cache = Arc::new( + OverflowLRUCache::::new(capacity, test_store).expect("should create cache"), + ); + + let mut pending_blocks = VecDeque::new(); + let mut pending_blobs = VecDeque::new(); + let mut roots = VecDeque::new(); + let mut epoch_count = BTreeMap::new(); + while pending_blobs.len() < n_epochs * capacity { + let (pending_block, blobs) = availability_pending_block(&harness, log.clone()).await; + if pending_block.num_blobs_expected() == 0 { + // we need blocks with blobs + continue; + } + let root = pending_block.block.as_block().canonical_root(); + let epoch = pending_block + .block + .as_block() + .slot() + .epoch(E::slots_per_epoch()); + epoch_count.entry(epoch).or_insert_with(|| 0).add_assign(1); + + pending_blocks.push_back(pending_block); + pending_blobs.push_back(blobs); + roots.push_back(root); + } + + let kzg = harness + .chain + .kzg + .as_ref() + .cloned() + .expect("kzg should exist"); + + for _ in 0..(n_epochs * capacity) { + let pending_block = pending_blocks.pop_front().expect("should have block"); + let expected_blobs = pending_block.num_blobs_expected(); + if expected_blobs > 1 { + // might as well add a blob too + let mut pending_blobs = pending_blobs.pop_front().expect("should have blobs"); + let one_blob = pending_blobs.pop().expect("should have at least one blob"); + let kzg_verified_blob = + verify_kzg_for_blob(one_blob, kzg.as_ref()).expect("kzg should verify"); + // generate random boolean + let block_first = (rand::random::() % 2) == 0; + if block_first { + let availability = cache + .put_pending_executed_block(pending_block) + .expect("should put block"); + assert!( + matches!(availability, Availability::PendingBlobs(_)), + "should have pending blobs" + ); + let availability = cache + .put_kzg_verified_blob(kzg_verified_blob) + .expect("should put blob"); + assert!( + matches!(availability, Availability::PendingBlobs(_)), + "availabilty should be pending blobs: {:?}", + availability + ); + } else { + let availability = cache + .put_kzg_verified_blob(kzg_verified_blob) + .expect("should put blob"); + let root = pending_block.block.as_block().canonical_root(); + assert_eq!( + availability, + Availability::PendingBlock(root), + "should be pending block" + ); + let availability = cache + .put_pending_executed_block(pending_block) + .expect("should put block"); + assert!( + matches!(availability, Availability::PendingBlobs(_)), + "should have pending blobs" + ); + } + } else { + // still need to pop front so the blob count is correct + pending_blobs.pop_front().expect("should have blobs"); + let availability = cache + .put_pending_executed_block(pending_block) + .expect("should put block"); + assert!( + matches!(availability, Availability::PendingBlobs(_)), + "should be pending blobs" + ); + } + } + + // now we should have a full cache spanning multiple epochs + // run the maintenance routine for increasing epochs and ensure that the cache is pruned + assert_eq!( + cache.critical.read().in_memory.len(), + capacity, + "cache memory should be full" + ); + let store_keys = cache + .overflow_store + .read_keys_on_disk() + .expect("should read keys"); + assert_eq!( + store_keys.len(), + capacity * (n_epochs - 1), + "cache disk should have the rest" + ); + let mut expected_length = n_epochs * capacity; + for (epoch, count) in epoch_count { + cache + .do_maintenance(epoch + 1) + .expect("should run maintenance"); + let disk_keys = cache + .overflow_store + .read_keys_on_disk() + .expect("should read keys") + .len(); + let mem_keys = cache.critical.read().in_memory.len(); + expected_length -= count; + info!( + log, + "EPOCH: {} DISK KEYS: {} MEM KEYS: {} TOTAL: {} EXPECTED: {}", + epoch, + disk_keys, + mem_keys, + (disk_keys + mem_keys), + std::cmp::max(expected_length, capacity * 3 / 4), + ); + assert_eq!( + (disk_keys + mem_keys), + std::cmp::max(expected_length, capacity * 3 / 4), + "cache should be pruned" + ); + } + } + + #[tokio::test] + #[cfg(feature = "spec-minimal")] + async fn overflow_cache_test_persist_recover() { + type E = MinimalEthSpec; + type T = DiskHarnessType; + let log = test_logger(); + let chain_db_path = tempdir().expect("should get temp dir"); + let harness: BeaconChainHarness = get_deneb_chain(log.clone(), &chain_db_path).await; + let spec = harness.spec.clone(); + let n_epochs = 4; + let capacity = E::slots_per_epoch() as usize; + let db_path = tempdir().expect("should get temp dir"); + let test_store = get_store_with_spec::(&db_path, spec.clone(), log.clone()); + let cache = Arc::new( + OverflowLRUCache::::new(capacity, test_store.clone()).expect("should create cache"), + ); + + let mut pending_blocks = VecDeque::new(); + let mut pending_blobs = VecDeque::new(); + let mut roots = VecDeque::new(); + let mut epoch_count = BTreeMap::new(); + while pending_blobs.len() < n_epochs * capacity { + let (pending_block, blobs) = availability_pending_block(&harness, log.clone()).await; + if pending_block.num_blobs_expected() == 0 { + // we need blocks with blobs + continue; + } + let root = pending_block.block.as_block().canonical_root(); + let epoch = pending_block + .block + .as_block() + .slot() + .epoch(E::slots_per_epoch()); + epoch_count.entry(epoch).or_insert_with(|| 0).add_assign(1); + + pending_blocks.push_back(pending_block); + pending_blobs.push_back(blobs); + roots.push_back(root); + } + + let kzg = harness + .chain + .kzg + .as_ref() + .cloned() + .expect("kzg should exist"); + + let mut remaining_blobs = HashMap::new(); + for _ in 0..(n_epochs * capacity) { + let pending_block = pending_blocks.pop_front().expect("should have block"); + let block_root = pending_block.block.as_block().canonical_root(); + let expected_blobs = pending_block.num_blobs_expected(); + if expected_blobs > 1 { + // might as well add a blob too + let mut pending_blobs = pending_blobs.pop_front().expect("should have blobs"); + let one_blob = pending_blobs.pop().expect("should have at least one blob"); + let kzg_verified_blob = + verify_kzg_for_blob(one_blob, kzg.as_ref()).expect("kzg should verify"); + // generate random boolean + let block_first = (rand::random::() % 2) == 0; + remaining_blobs.insert(block_root, pending_blobs); + if block_first { + let availability = cache + .put_pending_executed_block(pending_block) + .expect("should put block"); + assert!( + matches!(availability, Availability::PendingBlobs(_)), + "should have pending blobs" + ); + let availability = cache + .put_kzg_verified_blob(kzg_verified_blob) + .expect("should put blob"); + assert!( + matches!(availability, Availability::PendingBlobs(_)), + "availabilty should be pending blobs: {:?}", + availability + ); + } else { + let availability = cache + .put_kzg_verified_blob(kzg_verified_blob) + .expect("should put blob"); + let root = pending_block.block.as_block().canonical_root(); + assert_eq!( + availability, + Availability::PendingBlock(root), + "should be pending block" + ); + let availability = cache + .put_pending_executed_block(pending_block) + .expect("should put block"); + assert!( + matches!(availability, Availability::PendingBlobs(_)), + "should have pending blobs" + ); + } + } else { + // still need to pop front so the blob count is correct + let pending_blobs = pending_blobs.pop_front().expect("should have blobs"); + remaining_blobs.insert(block_root, pending_blobs); + let availability = cache + .put_pending_executed_block(pending_block) + .expect("should put block"); + assert!( + matches!(availability, Availability::PendingBlobs(_)), + "should be pending blobs" + ); + } + } + + // now we should have a full cache spanning multiple epochs + // cache should be at capacity + assert_eq!( + cache.critical.read().in_memory.len(), + capacity, + "cache memory should be full" + ); + // write all components to disk + cache.write_all_to_disk().expect("should write all to disk"); + // everything should be on disk now + assert_eq!( + cache + .overflow_store + .read_keys_on_disk() + .expect("should read keys") + .len(), + capacity * n_epochs, + "cache disk should have the rest" + ); + assert_eq!( + cache.critical.read().in_memory.len(), + 0, + "cache memory should be empty" + ); + assert_eq!( + cache.critical.read().store_keys.len(), + n_epochs * capacity, + "cache store should have the rest" + ); + drop(cache); + + // create a new cache with the same store + let recovered_cache = + OverflowLRUCache::::new(capacity, test_store).expect("should recover cache"); + // again, everything should be on disk + assert_eq!( + recovered_cache + .overflow_store + .read_keys_on_disk() + .expect("should read keys") + .len(), + capacity * n_epochs, + "cache disk should have the rest" + ); + assert_eq!( + recovered_cache.critical.read().in_memory.len(), + 0, + "cache memory should be empty" + ); + assert_eq!( + recovered_cache.critical.read().store_keys.len(), + n_epochs * capacity, + "cache store should have the rest" + ); + + // now lets insert the remaining blobs until the cache is empty + for (_, blobs) in remaining_blobs { + let additional_blobs = blobs.len(); + for (i, gossip_blob) in blobs.into_iter().enumerate() { + let kzg_verified_blob = + verify_kzg_for_blob(gossip_blob, kzg.as_ref()).expect("kzg should verify"); + let availability = recovered_cache + .put_kzg_verified_blob(kzg_verified_blob) + .expect("should put blob"); + if i == additional_blobs - 1 { + assert!(matches!(availability, Availability::Available(_))) + } else { + assert!(matches!(availability, Availability::PendingBlobs(_))); + } + } + } + } +} diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 9d5485df9ed..e4c4ff2517c 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -2,6 +2,7 @@ use crate::attester_cache::Error as AttesterCacheError; use crate::beacon_block_streamer::Error as BlockStreamerError; use crate::beacon_chain::ForkChoiceError; use crate::beacon_fork_choice_store::Error as ForkChoiceStoreError; +use crate::data_availability_checker::AvailabilityCheckError; use crate::eth1_chain::Error as Eth1ChainError; use crate::historical_blocks::HistoricalBlockError; use crate::migrate::PruningError; @@ -215,6 +216,7 @@ pub enum BeaconChainError { BlsToExecutionConflictsWithPool, InconsistentFork(InconsistentFork), ProposerHeadForkChoiceError(fork_choice::Error), + AvailabilityCheckError(AvailabilityCheckError), } easy_from_to!(SlotProcessingError, BeaconChainError); @@ -240,6 +242,7 @@ easy_from_to!(HistoricalBlockError, BeaconChainError); easy_from_to!(StateAdvanceError, BeaconChainError); easy_from_to!(BlockReplayError, BeaconChainError); easy_from_to!(InconsistentFork, BeaconChainError); +easy_from_to!(AvailabilityCheckError, BeaconChainError); #[derive(Debug)] pub enum BlockProductionError { diff --git a/beacon_node/beacon_chain/src/eth1_finalization_cache.rs b/beacon_node/beacon_chain/src/eth1_finalization_cache.rs index 7cf805a126d..17ac4e5b30b 100644 --- a/beacon_node/beacon_chain/src/eth1_finalization_cache.rs +++ b/beacon_node/beacon_chain/src/eth1_finalization_cache.rs @@ -1,4 +1,5 @@ use slog::{debug, Logger}; +use ssz_derive::{Decode, Encode}; use std::cmp; use std::collections::BTreeMap; use types::{Checkpoint, Epoch, Eth1Data, Hash256 as Root}; @@ -10,7 +11,7 @@ pub const DEFAULT_ETH1_CACHE_SIZE: usize = 5; /// These fields are named the same as the corresponding fields in the `BeaconState` /// as this structure stores these values from the `BeaconState` at a `Checkpoint` -#[derive(Clone)] +#[derive(Clone, Debug, PartialEq, Encode, Decode)] pub struct Eth1FinalizationData { pub eth1_data: Eth1Data, pub eth1_deposit_index: u64, diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 315f869514b..a8fdc0abd67 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -380,6 +380,8 @@ lazy_static! { try_create_histogram("beacon_persist_eth1_cache", "Time taken to persist the eth1 caches"); pub static ref PERSIST_FORK_CHOICE: Result = try_create_histogram("beacon_persist_fork_choice", "Time taken to persist the fork choice struct"); + pub static ref PERSIST_DATA_AVAILABILITY_CHECKER: Result = + try_create_histogram("beacon_persist_data_availability_checker", "Time taken to persist the data availability checker"); /* * Eth1 diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 329f0727542..c977746c7b0 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -2,6 +2,7 @@ use crate::address_change_broadcast::broadcast_address_changes_at_capella; use crate::config::{ClientGenesis, Config as ClientConfig}; use crate::notifier::spawn_notifier; use crate::Client; +use beacon_chain::data_availability_checker::start_availability_cache_maintenance_service; use beacon_chain::otb_verification_service::start_otb_verification_service; use beacon_chain::proposer_prep_service::start_proposer_prep_service; use beacon_chain::schema_change::migrate_schema; @@ -828,6 +829,10 @@ where start_proposer_prep_service(runtime_context.executor.clone(), beacon_chain.clone()); start_otb_verification_service(runtime_context.executor.clone(), beacon_chain.clone()); + start_availability_cache_maintenance_service( + runtime_context.executor.clone(), + beacon_chain.clone(), + ); } Ok(Client { diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 773c3fe9d4e..5e5508b6f8e 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use std::sync::Arc; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; +use types::consts::deneb::BLOB_TX_TYPE; use types::transaction::{BlobTransaction, EcdsaSignature, SignedBlobTransaction}; use types::{ Blob, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, @@ -684,7 +685,7 @@ impl ExecutionBlockGenerator { signature: bad_signature, }; // calculate transaction bytes - let tx_bytes = [0x05u8] + let tx_bytes = [BLOB_TX_TYPE] .into_iter() .chain(signed_blob_transaction.as_ssz_bytes().into_iter()) .collect::>(); diff --git a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs index 5f282ecfbee..ffaf5641278 100644 --- a/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs +++ b/beacon_node/network/src/beacon_processor/worker/rpc_methods.rs @@ -230,7 +230,7 @@ impl Worker { let mut blob_list_results = HashMap::new(); for id in request.blob_ids.into_iter() { // First attempt to get the blobs from the RPC cache. - if let Some(blob) = self.chain.data_availability_checker.get_blob(&id) { + if let Ok(Some(blob)) = self.chain.data_availability_checker.get_blob(&id) { self.send_response(peer_id, Response::BlobsByRoot(Some(blob)), request_id); send_blob_count += 1; } else { diff --git a/beacon_node/store/src/leveldb_store.rs b/beacon_node/store/src/leveldb_store.rs index 86bd4ffaccd..261f8c461b0 100644 --- a/beacon_node/store/src/leveldb_store.rs +++ b/beacon_node/store/src/leveldb_store.rs @@ -198,6 +198,36 @@ impl KeyValueStore for LevelDB { ) } + fn iter_raw_entries(&self, column: DBColumn, prefix: &[u8]) -> RawEntryIter { + let start_key = BytesKey::from_vec(get_key_for_col(column.into(), prefix)); + + let iter = self.db.iter(self.read_options()); + iter.seek(&start_key); + + Box::new( + iter.take_while(move |(key, _)| key.key.starts_with(start_key.key.as_slice())) + .map(move |(bytes_key, value)| { + let subkey = &bytes_key.key[column.as_bytes().len()..]; + Ok((Vec::from(subkey), value)) + }), + ) + } + + fn iter_raw_keys(&self, column: DBColumn, prefix: &[u8]) -> RawKeyIter { + let start_key = BytesKey::from_vec(get_key_for_col(column.into(), prefix)); + + let iter = self.db.keys_iter(self.read_options()); + iter.seek(&start_key); + + Box::new( + iter.take_while(move |key| key.key.starts_with(start_key.key.as_slice())) + .map(move |bytes_key| { + let subkey = &bytes_key.key[column.as_bytes().len()..]; + Ok(Vec::from(subkey)) + }), + ) + } + /// Iterate through all keys and values in a particular column. fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter { let start_key = diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 47f0049fc2b..cd2f2da2b95 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -49,6 +49,9 @@ pub use types::*; pub type ColumnIter<'a> = Box), Error>> + 'a>; pub type ColumnKeyIter<'a> = Box> + 'a>; +pub type RawEntryIter<'a> = Box, Vec), Error>> + 'a>; +pub type RawKeyIter<'a> = Box, Error>> + 'a>; + pub trait KeyValueStore: Sync + Send + Sized + 'static { /// Retrieve some bytes in `column` with `key`. fn get_bytes(&self, column: &str, key: &[u8]) -> Result>, Error>; @@ -88,6 +91,14 @@ pub trait KeyValueStore: Sync + Send + Sized + 'static { Box::new(std::iter::empty()) } + fn iter_raw_entries(&self, _column: DBColumn, _prefix: &[u8]) -> RawEntryIter { + Box::new(std::iter::empty()) + } + + fn iter_raw_keys(&self, _column: DBColumn, _prefix: &[u8]) -> RawKeyIter { + Box::new(std::iter::empty()) + } + /// Iterate through all keys in a particular column. fn iter_column_keys(&self, _column: DBColumn) -> ColumnKeyIter { // Default impl for non LevelDB databases @@ -227,6 +238,8 @@ pub enum DBColumn { OptimisticTransitionBlock, #[strum(serialize = "bhs")] BeaconHistoricalSummaries, + #[strum(serialize = "olc")] + OverflowLRUCache, } /// A block from the database, which might have an execution payload or not. diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 03942751a86..b78e486d512 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -192,7 +192,8 @@ impl CountUnrealized { /// Indicates if a block has been verified by an execution payload. /// /// There is no variant for "invalid", since such a block should never be added to fork choice. -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Encode, Decode)] +#[ssz(enum_behaviour = "tag")] pub enum PayloadVerificationStatus { /// An EL has declared the execution payload to be valid. Verified, diff --git a/consensus/ssz_derive/src/lib.rs b/consensus/ssz_derive/src/lib.rs index a5c4b7bb6dd..280bdb83df8 100644 --- a/consensus/ssz_derive/src/lib.rs +++ b/consensus/ssz_derive/src/lib.rs @@ -4,6 +4,7 @@ //! //! The following struct/enum attributes are available: //! +//! - `#[ssz(enum_behaviour = "tag")]`: encodes and decodes an `enum` with 0 fields per variant //! - `#[ssz(enum_behaviour = "union")]`: encodes and decodes an `enum` with a one-byte variant selector. //! - `#[ssz(enum_behaviour = "transparent")]`: allows encoding an `enum` by serializing only the //! value whilst ignoring outermost the `enum`. @@ -140,6 +141,22 @@ //! TransparentEnum::Bar(vec![42, 42]).as_ssz_bytes(), //! vec![42, 42] //! ); +//! +//! /// Representated as an SSZ "uint8" +//! #[derive(Debug, PartialEq, Encode, Decode)] +//! #[ssz(enum_behaviour = "tag")] +//! enum TagEnum { +//! Foo, +//! Bar, +//! } +//! assert_eq!( +//! TagEnum::Foo.as_ssz_bytes(), +//! vec![0] +//! ); +//! assert_eq!( +//! TagEnum::from_ssz_bytes(&[1]).unwrap(), +//! TagEnum::Bar, +//! ); //! ``` use darling::{FromDeriveInput, FromMeta}; @@ -154,8 +171,9 @@ const MAX_UNION_SELECTOR: u8 = 127; const ENUM_TRANSPARENT: &str = "transparent"; const ENUM_UNION: &str = "union"; +const ENUM_TAG: &str = "tag"; const NO_ENUM_BEHAVIOUR_ERROR: &str = "enums require an \"enum_behaviour\" attribute with \ - a \"transparent\" or \"union\" value, e.g., #[ssz(enum_behaviour = \"transparent\")]"; + a \"transparent\", \"union\", or \"tag\" value, e.g., #[ssz(enum_behaviour = \"transparent\")]"; #[derive(Debug, FromDeriveInput)] #[darling(attributes(ssz))] @@ -196,6 +214,7 @@ enum StructBehaviour { enum EnumBehaviour { Union, Transparent, + Tag, } impl<'a> Procedure<'a> { @@ -237,6 +256,10 @@ impl<'a> Procedure<'a> { data, behaviour: EnumBehaviour::Transparent, }, + Some("tag") => Procedure::Enum { + data, + behaviour: EnumBehaviour::Tag, + }, Some(other) => panic!( "{} is not a valid enum behaviour, use \"container\" or \"transparent\"", other @@ -296,6 +319,7 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { Procedure::Enum { data, behaviour } => match behaviour { EnumBehaviour::Transparent => ssz_encode_derive_enum_transparent(&item, data), EnumBehaviour::Union => ssz_encode_derive_enum_union(&item, data), + EnumBehaviour::Tag => ssz_encode_derive_enum_tag(&item, data), }, } } @@ -573,6 +597,67 @@ fn ssz_encode_derive_enum_transparent( output.into() } +/// Derive `ssz::Encode` for an `enum` following the "tag" method. +/// +/// The union selector will be determined based upon the order in which the enum variants are +/// defined. E.g., the top-most variant in the enum will have a selector of `0`, the variant +/// beneath it will have a selector of `1` and so on. +/// +/// # Limitations +/// +/// Only supports enums where each variant has no fields +fn ssz_encode_derive_enum_tag(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream { + let name = &derive_input.ident; + let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl(); + + let patterns: Vec<_> = enum_data + .variants + .iter() + .map(|variant| { + let variant_name = &variant.ident; + + if !variant.fields.is_empty() { + panic!("ssz::Encode tag behaviour can only be derived for enums with no fields"); + } + + quote! { + #name::#variant_name + } + }) + .collect(); + + let union_selectors = compute_union_selectors(patterns.len()); + + let output = quote! { + impl #impl_generics ssz::Encode for #name #ty_generics #where_clause { + fn is_ssz_fixed_len() -> bool { + true + } + + fn ssz_fixed_len() -> usize { + 1 + } + + fn ssz_bytes_len(&self) -> usize { + 1 + } + + fn ssz_append(&self, buf: &mut Vec) { + match self { + #( + #patterns => { + let union_selector: u8 = #union_selectors; + debug_assert!(union_selector <= ssz::MAX_UNION_SELECTOR); + buf.push(union_selector); + }, + )* + } + } + } + }; + output.into() +} + /// Derive `ssz::Encode` for an `enum` following the "union" SSZ spec. /// /// The union selector will be determined based upon the order in which the enum variants are @@ -652,9 +737,10 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream { }, Procedure::Enum { data, behaviour } => match behaviour { EnumBehaviour::Union => ssz_decode_derive_enum_union(&item, data), + EnumBehaviour::Tag => ssz_decode_derive_enum_tag(&item, data), EnumBehaviour::Transparent => panic!( - "Decode cannot be derived for enum_behaviour \"{}\", only \"{}\" is valid.", - ENUM_TRANSPARENT, ENUM_UNION + "Decode cannot be derived for enum_behaviour \"{}\", only \"{}\" and \"{}\" is valid.", + ENUM_TRANSPARENT, ENUM_UNION, ENUM_TAG, ), }, } @@ -908,6 +994,59 @@ fn ssz_decode_derive_struct_transparent( output.into() } +/// Derive `ssz::Decode` for an `enum` following the "tag" SSZ spec. +fn ssz_decode_derive_enum_tag(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream { + let name = &derive_input.ident; + let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl(); + + let patterns: Vec<_> = enum_data + .variants + .iter() + .map(|variant| { + let variant_name = &variant.ident; + + if !variant.fields.is_empty() { + panic!("ssz::Decode tag behaviour can only be derived for enums with no fields"); + } + + quote! { + #name::#variant_name + } + }) + .collect(); + + let union_selectors = compute_union_selectors(patterns.len()); + + let output = quote! { + impl #impl_generics ssz::Decode for #name #ty_generics #where_clause { + fn is_ssz_fixed_len() -> bool { + true + } + + fn ssz_fixed_len() -> usize { + 1 + } + + fn from_ssz_bytes(bytes: &[u8]) -> std::result::Result { + let byte = bytes + .first() + .copied() + .ok_or(ssz::DecodeError::OutOfBoundsByte { i: 0 })?; + + match byte { + #( + #union_selectors => { + Ok(#patterns) + }, + )* + other => Err(ssz::DecodeError::UnionSelectorInvalid(other)), + } + } + } + }; + output.into() +} + /// Derive `ssz::Decode` for an `enum` following the "union" SSZ spec. fn ssz_decode_derive_enum_union(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream { let name = &derive_input.ident; diff --git a/consensus/ssz_derive/tests/tests.rs b/consensus/ssz_derive/tests/tests.rs index 040d2a34761..72192b293a1 100644 --- a/consensus/ssz_derive/tests/tests.rs +++ b/consensus/ssz_derive/tests/tests.rs @@ -12,6 +12,14 @@ fn assert_encode_decode(item: &T, bytes: assert_eq!(T::from_ssz_bytes(bytes).unwrap(), *item); } +#[derive(PartialEq, Debug, Encode, Decode)] +#[ssz(enum_behaviour = "tag")] +enum TagEnum { + A, + B, + C, +} + #[derive(PartialEq, Debug, Encode, Decode)] #[ssz(enum_behaviour = "union")] enum TwoFixedUnion { @@ -120,6 +128,13 @@ fn two_variable_union() { ); } +#[test] +fn tag_enum() { + assert_encode_decode(&TagEnum::A, &[0]); + assert_encode_decode(&TagEnum::B, &[1]); + assert_encode_decode(&TagEnum::C, &[2]); +} + #[derive(PartialEq, Debug, Encode, Decode)] #[ssz(enum_behaviour = "union")] enum TwoVecUnion { diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index 37bd5fe446d..78803ab4eb4 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -1,5 +1,6 @@ use crate::common::get_indexed_attestation; use crate::per_block_processing::errors::{AttestationInvalid, BlockOperationError}; +use ssz_derive::{Decode, Encode}; use std::collections::{hash_map::Entry, HashMap}; use tree_hash::TreeHash; use types::{ @@ -7,7 +8,7 @@ use types::{ ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, SignedBeaconBlock, Slot, }; -#[derive(Debug, Clone)] +#[derive(Debug, PartialEq, Clone, Encode, Decode)] pub struct ConsensusContext { /// Slot to act as an identifier/safeguard slot: Slot, @@ -16,6 +17,8 @@ pub struct ConsensusContext { /// Block root of the block at `slot`. current_block_root: Option, /// Cache of indexed attestations constructed during block processing. + /// We can skip serializing / deserializing this as the cache will just be rebuilt + #[ssz(skip_serializing, skip_deserializing)] indexed_attestations: HashMap<(AttestationData, BitList), IndexedAttestation>, /// Whether `verify_kzg_commitments_against_transactions` has successfully passed. diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 27f15c9ed07..090a361cd47 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -201,13 +201,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockRef<'a, T, Payl /// dictated by `self.slot()`. pub fn fork_name(&self, spec: &ChainSpec) -> Result { let fork_at_slot = spec.fork_name_at_slot::(self.slot()); - let object_fork = match self { - BeaconBlockRef::Base { .. } => ForkName::Base, - BeaconBlockRef::Altair { .. } => ForkName::Altair, - BeaconBlockRef::Merge { .. } => ForkName::Merge, - BeaconBlockRef::Capella { .. } => ForkName::Capella, - BeaconBlockRef::Deneb { .. } => ForkName::Deneb, - }; + let object_fork = self.fork_name_unchecked(); if fork_at_slot == object_fork { Ok(object_fork) @@ -219,6 +213,19 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockRef<'a, T, Payl } } + /// Returns the name of the fork pertaining to `self`. + /// + /// Does not check that the fork is consistent with the slot. + pub fn fork_name_unchecked(&self) -> ForkName { + match self { + BeaconBlockRef::Base { .. } => ForkName::Base, + BeaconBlockRef::Altair { .. } => ForkName::Altair, + BeaconBlockRef::Merge { .. } => ForkName::Merge, + BeaconBlockRef::Capella { .. } => ForkName::Capella, + BeaconBlockRef::Deneb { .. } => ForkName::Deneb, + } + } + /// Convenience accessor for the `body` as a `BeaconBlockBodyRef`. pub fn body(&self) -> BeaconBlockBodyRef<'a, T, Payload> { map_beacon_block_ref_into_beacon_block_body_ref!(&'a _, *self, |block, cons| cons( diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index d480c0fc32e..58c0eed3398 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -415,13 +415,7 @@ impl BeaconState { /// dictated by `self.slot()`. pub fn fork_name(&self, spec: &ChainSpec) -> Result { let fork_at_slot = spec.fork_name_at_epoch(self.current_epoch()); - let object_fork = match self { - BeaconState::Base { .. } => ForkName::Base, - BeaconState::Altair { .. } => ForkName::Altair, - BeaconState::Merge { .. } => ForkName::Merge, - BeaconState::Capella { .. } => ForkName::Capella, - BeaconState::Deneb { .. } => ForkName::Deneb, - }; + let object_fork = self.fork_name_unchecked(); if fork_at_slot == object_fork { Ok(object_fork) @@ -433,6 +427,19 @@ impl BeaconState { } } + /// Returns the name of the fork pertaining to `self`. + /// + /// Does not check if `self` is consistent with the fork dictated by `self.slot()`. + pub fn fork_name_unchecked(&self) -> ForkName { + match self { + BeaconState::Base { .. } => ForkName::Base, + BeaconState::Altair { .. } => ForkName::Altair, + BeaconState::Merge { .. } => ForkName::Merge, + BeaconState::Capella { .. } => ForkName::Capella, + BeaconState::Deneb { .. } => ForkName::Deneb, + } + } + /// Specialised deserialisation method that uses the `ChainSpec` as context. #[allow(clippy::integer_arithmetic)] pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result { @@ -1870,3 +1877,80 @@ impl ForkVersionDeserialize for BeaconState { )) } } + +/// This module can be used to encode and decode a `BeaconState` the same way it +/// would be done if we had tagged the superstruct enum with +/// `#[ssz(enum_behaviour = "union")]` +/// This should _only_ be used for *some* cases to store these objects in the +/// database and _NEVER_ for encoding / decoding states sent over the network! +pub mod ssz_tagged_beacon_state { + use super::*; + pub mod encode { + use super::*; + #[allow(unused_imports)] + use ssz::*; + + pub fn is_ssz_fixed_len() -> bool { + false + } + + pub fn ssz_fixed_len() -> usize { + BYTES_PER_LENGTH_OFFSET + } + + pub fn ssz_bytes_len(state: &BeaconState) -> usize { + state + .ssz_bytes_len() + .checked_add(1) + .expect("encoded length must be less than usize::max") + } + + pub fn ssz_append(state: &BeaconState, buf: &mut Vec) { + let fork_name = state.fork_name_unchecked(); + fork_name.ssz_append(buf); + state.ssz_append(buf); + } + + pub fn as_ssz_bytes(state: &BeaconState) -> Vec { + let mut buf = vec![]; + ssz_append(state, &mut buf); + + buf + } + } + + pub mod decode { + use super::*; + #[allow(unused_imports)] + use ssz::*; + + pub fn is_ssz_fixed_len() -> bool { + false + } + + pub fn ssz_fixed_len() -> usize { + BYTES_PER_LENGTH_OFFSET + } + + pub fn from_ssz_bytes(bytes: &[u8]) -> Result, DecodeError> { + let fork_byte = bytes + .first() + .copied() + .ok_or(DecodeError::OutOfBoundsByte { i: 0 })?; + let body = bytes + .get(1..) + .ok_or(DecodeError::OutOfBoundsByte { i: 1 })?; + match ForkName::from_ssz_bytes(&[fork_byte])? { + ForkName::Base => Ok(BeaconState::Base(BeaconStateBase::from_ssz_bytes(body)?)), + ForkName::Altair => Ok(BeaconState::Altair(BeaconStateAltair::from_ssz_bytes( + body, + )?)), + ForkName::Merge => Ok(BeaconState::Merge(BeaconStateMerge::from_ssz_bytes(body)?)), + ForkName::Capella => Ok(BeaconState::Capella(BeaconStateCapella::from_ssz_bytes( + body, + )?)), + ForkName::Deneb => Ok(BeaconState::Deneb(BeaconStateDeneb::from_ssz_bytes(body)?)), + } + } + } +} diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index e7c1f9628bc..6d52e0abbd6 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -1,12 +1,14 @@ use crate::{ChainSpec, Epoch}; use serde_derive::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; use std::convert::TryFrom; use std::fmt::{self, Display, Formatter}; use std::str::FromStr; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Decode, Encode, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(try_from = "String")] #[serde(into = "String")] +#[ssz(enum_behaviour = "tag")] pub enum ForkName { Base, Altair, diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 617cbcaf02d..46c5c2a4ce8 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -170,9 +170,9 @@ pub use crate::selection_proof::SelectionProof; pub use crate::shuffling_id::AttestationShufflingId; pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof; pub use crate::signed_beacon_block::{ - SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, - SignedBeaconBlockDeneb, SignedBeaconBlockHash, SignedBeaconBlockMerge, - SignedBlindedBeaconBlock, + ssz_tagged_signed_beacon_block, SignedBeaconBlock, SignedBeaconBlockAltair, + SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockHash, + SignedBeaconBlockMerge, SignedBlindedBeaconBlock, }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_blob::*; diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index 58810150c21..23a254079d2 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -92,6 +92,12 @@ impl> SignedBeaconBlock self.message().fork_name(spec) } + /// Returns the name of the fork pertaining to `self` + /// Does not check that the fork is consistent with the slot. + pub fn fork_name_unchecked(&self) -> ForkName { + self.message().fork_name_unchecked() + } + /// SSZ decode with fork variant determined by slot. pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result { Self::from_ssz_bytes_with(bytes, |bytes| BeaconBlock::from_ssz_bytes(bytes, spec)) @@ -510,6 +516,99 @@ impl> ForkVersionDeserialize } } +/// This module can be used to encode and decode a `SignedBeaconBlock` the same way it +/// would be done if we had tagged the superstruct enum with +/// `#[ssz(enum_behaviour = "union")]` +/// This should _only_ be used *some* cases when storing these objects in the database +/// and _NEVER_ for encoding / decoding blocks sent over the network! +pub mod ssz_tagged_signed_beacon_block { + use super::*; + pub mod encode { + use super::*; + #[allow(unused_imports)] + use ssz::*; + + pub fn is_ssz_fixed_len() -> bool { + false + } + + pub fn ssz_fixed_len() -> usize { + BYTES_PER_LENGTH_OFFSET + } + + pub fn ssz_bytes_len>( + block: &SignedBeaconBlock, + ) -> usize { + block + .ssz_bytes_len() + .checked_add(1) + .expect("encoded length must be less than usize::max") + } + + pub fn ssz_append>( + block: &SignedBeaconBlock, + buf: &mut Vec, + ) { + let fork_name = block.fork_name_unchecked(); + fork_name.ssz_append(buf); + block.ssz_append(buf); + } + + pub fn as_ssz_bytes>( + block: &SignedBeaconBlock, + ) -> Vec { + let mut buf = vec![]; + ssz_append(block, &mut buf); + + buf + } + } + + pub mod decode { + use super::*; + #[allow(unused_imports)] + use ssz::*; + + pub fn is_ssz_fixed_len() -> bool { + false + } + + pub fn ssz_fixed_len() -> usize { + BYTES_PER_LENGTH_OFFSET + } + + pub fn from_ssz_bytes>( + bytes: &[u8], + ) -> Result, DecodeError> { + let fork_byte = bytes + .first() + .copied() + .ok_or(DecodeError::OutOfBoundsByte { i: 0 })?; + let body = bytes + .get(1..) + .ok_or(DecodeError::OutOfBoundsByte { i: 1 })?; + + match ForkName::from_ssz_bytes(&[fork_byte])? { + ForkName::Base => Ok(SignedBeaconBlock::Base( + SignedBeaconBlockBase::from_ssz_bytes(body)?, + )), + ForkName::Altair => Ok(SignedBeaconBlock::Altair( + SignedBeaconBlockAltair::from_ssz_bytes(body)?, + )), + ForkName::Merge => Ok(SignedBeaconBlock::Merge( + SignedBeaconBlockMerge::from_ssz_bytes(body)?, + )), + ForkName::Capella => Ok(SignedBeaconBlock::Capella( + SignedBeaconBlockCapella::from_ssz_bytes(body)?, + )), + ForkName::Deneb => Ok(SignedBeaconBlock::Deneb( + SignedBeaconBlockDeneb::from_ssz_bytes(body)?, + )), + } + } + } +} + #[cfg(test)] mod test { use super::*; @@ -551,4 +650,38 @@ mod test { assert_eq!(reconstructed, block); } } + + #[test] + fn test_ssz_tagged_signed_beacon_block() { + type E = MainnetEthSpec; + + let spec = &E::default_spec(); + let sig = Signature::empty(); + let blocks = vec![ + SignedBeaconBlock::::from_block( + BeaconBlock::Base(BeaconBlockBase::empty(spec)), + sig.clone(), + ), + SignedBeaconBlock::from_block( + BeaconBlock::Altair(BeaconBlockAltair::empty(spec)), + sig.clone(), + ), + SignedBeaconBlock::from_block( + BeaconBlock::Merge(BeaconBlockMerge::empty(spec)), + sig.clone(), + ), + SignedBeaconBlock::from_block( + BeaconBlock::Capella(BeaconBlockCapella::empty(spec)), + sig.clone(), + ), + SignedBeaconBlock::from_block(BeaconBlock::Deneb(BeaconBlockDeneb::empty(spec)), sig), + ]; + + for block in blocks { + let encoded = ssz_tagged_signed_beacon_block::encode::as_ssz_bytes(&block); + let decoded = ssz_tagged_signed_beacon_block::decode::from_ssz_bytes::(&encoded) + .expect("should decode"); + assert_eq!(decoded, block); + } + } } From 8f3e8f50f83329b7249051806594dcd4cc5ba23d Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Mon, 15 May 2023 09:54:48 +0200 Subject: [PATCH 93/96] minor fixes --- .../process_operations.rs | 22 +++++++++---------- .../ef_tests/src/cases/epoch_processing.rs | 6 +---- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 558554ed6fb..a13131c11c4 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -16,22 +16,20 @@ pub fn process_operations>( ctxt: &mut ConsensusContext, spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { - let unprocessed_deposits_count = state - .eth1_data() - .deposit_count - .saturating_sub(state.eth1_deposit_index()); - let max_deposits = ::MaxDeposits::to_u64(); let eth1_deposit_index_limit = state .deposit_receipts_start_index() .unwrap_or_else(|_| state.eth1_data().deposit_count); - if state.eth1_deposit_index() < eth1_deposit_index_limit { - if block_body.deposits().len() - != std::cmp::min(max_deposits as usize, unprocessed_deposits_count as usize) - { - return Err(BlockProcessingError::DepositReceiptError); - } - } else if !block_body.deposits().is_empty() { + let expected_deposit_count = if state.eth1_deposit_index() < eth1_deposit_index_limit { + std::cmp::min( + ::MaxDeposits::to_u64() as usize, + (eth1_deposit_index_limit - state.eth1_deposit_index()) as usize, + ) as u64 + } else { + 0 + }; + + if block_body.deposits().len() as u64 != expected_deposit_count { return Err(BlockProcessingError::DepositReceiptError); } diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index adbb890af08..34c192063ad 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -320,11 +320,7 @@ impl> Case for EpochProcessing { T::name() != "participation_record_updates" && T::name() != "historical_summaries_update" } - ForkName::Capella | ForkName::Eip4844 => { - T::name() != "participation_record_updates" - && T::name() != "historical_roots_update" - } - ForkName::Eip6110 => { + ForkName::Capella | ForkName::Eip4844 | ForkName::Eip6110 => { T::name() != "participation_record_updates" && T::name() != "historical_roots_update" } From c4b2f1c8ac132db8220f8767fa58846f1ba40470 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 15 May 2023 12:37:51 -0400 Subject: [PATCH 94/96] fix count usage in blobs by range (#4289) --- beacon_node/lighthouse_network/src/rpc/outbound.rs | 2 +- beacon_node/lighthouse_network/src/rpc/protocol.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/lighthouse_network/src/rpc/outbound.rs b/beacon_node/lighthouse_network/src/rpc/outbound.rs index 6115f874efb..a75ec125011 100644 --- a/beacon_node/lighthouse_network/src/rpc/outbound.rs +++ b/beacon_node/lighthouse_network/src/rpc/outbound.rs @@ -112,7 +112,7 @@ impl OutboundRequest { OutboundRequest::Goodbye(_) => 0, OutboundRequest::BlocksByRange(req) => req.count, OutboundRequest::BlocksByRoot(req) => req.block_roots.len() as u64, - OutboundRequest::BlobsByRange(req) => req.count, + OutboundRequest::BlobsByRange(req) => req.count * TSpec::max_blobs_per_block() as u64, OutboundRequest::BlobsByRoot(req) => req.blob_ids.len() as u64, OutboundRequest::Ping(_) => 1, OutboundRequest::MetaData(_) => 1, diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index ed3b5a779fe..ea6293cf9b2 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -528,7 +528,7 @@ impl InboundRequest { InboundRequest::Goodbye(_) => 0, InboundRequest::BlocksByRange(req) => req.count, InboundRequest::BlocksByRoot(req) => req.block_roots.len() as u64, - InboundRequest::BlobsByRange(req) => req.count, + InboundRequest::BlobsByRange(req) => req.count * TSpec::max_blobs_per_block() as u64, InboundRequest::BlobsByRoot(req) => req.blob_ids.len() as u64, InboundRequest::Ping(_) => 1, InboundRequest::MetaData(_) => 1, From 9b55d74c1cbff5db007a5b487480763da93f4537 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Tue, 16 May 2023 09:43:26 -0400 Subject: [PATCH 95/96] fix db startup (#4298) --- beacon_node/store/src/hot_cold_store.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 8071f3eea2d..a4cf263f1aa 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -260,8 +260,7 @@ impl HotColdDB, LevelDB> { db.blobs_db = Some(LevelDB::open(path.as_path())?); } } - let blob_info = blob_info.unwrap_or_else(|| db.get_blob_info()); - db.compare_and_set_blob_info_with_write(blob_info, new_blob_info)?; + db.compare_and_set_blob_info_with_write(<_>::default(), new_blob_info)?; info!( db.log, "Blobs DB initialized"; From 6495fc1171276aaf9c068132d356c0c2dfa4cc83 Mon Sep 17 00:00:00 2001 From: kevinbogner Date: Thu, 18 May 2023 15:40:41 +0200 Subject: [PATCH 96/96] Merge deneb-free-blobs into eip6110-deneb --- beacon_node/beacon_chain/src/beacon_chain.rs | 3 +- beacon_node/beacon_chain/src/test_utils.rs | 2 +- beacon_node/execution_layer/src/engine_api.rs | 22 ++-- .../execution_layer/src/engine_api/http.rs | 102 ++++++++++-------- .../src/engine_api/json_structures.rs | 12 +-- beacon_node/execution_layer/src/lib.rs | 31 +----- .../test_utils/execution_block_generator.rs | 7 +- .../src/test_utils/handle_rpc.rs | 18 ++++ .../src/test_utils/mock_builder.rs | 12 +-- .../src/test_utils/mock_execution_layer.rs | 2 - .../execution_layer/src/test_utils/mod.rs | 4 +- .../http_api/src/build_block_contents.rs | 2 +- beacon_node/http_api/src/publish_blocks.rs | 2 +- .../src/rpc/codec/ssz_snappy.rs | 4 +- .../lighthouse_network/src/types/pubsub.rs | 11 +- .../store/src/impls/execution_payload.rs | 2 +- common/eth2/src/types.rs | 3 +- consensus/state_processing/src/genesis.rs | 4 +- .../state_processing/src/upgrade/eip6110.rs | 2 +- consensus/types/src/beacon_block.rs | 7 +- consensus/types/src/beacon_block_body.rs | 5 +- consensus/types/src/beacon_state.rs | 6 +- .../types/src/execution_payload_header.rs | 6 +- consensus/types/src/fork_name.rs | 2 +- consensus/types/src/lib.rs | 11 +- consensus/types/src/payload.rs | 14 +-- consensus/types/src/signed_beacon_block.rs | 3 + lcli/src/parse_ssz.rs | 2 +- testing/ef_tests/src/cases/operations.rs | 3 +- testing/ef_tests/src/cases/transition.rs | 2 +- 30 files changed, 162 insertions(+), 144 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 8e56c834b5a..bb7677dd302 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4861,7 +4861,7 @@ impl BeaconChain { ) } BeaconState::Eip6110(_) => { - let (payload, kzg_commitments, blobs) = block_contents + let (payload, kzg_commitments, blobs, proofs) = block_contents .ok_or(BlockProductionError::MissingExecutionPayload)? .deconstruct(); ( @@ -4890,6 +4890,7 @@ impl BeaconChain { }, }), blobs, + proofs, ) } }; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 22c50156a3c..9d740634000 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -824,7 +824,7 @@ where | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Merge(_) | SignedBeaconBlock::Capella(_) => (signed_block, None), - SignedBeaconBlock::Deneb(_) => { + SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Eip6110(_) => { if let Some(blobs) = self .chain .proposal_blob_cache diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 6dfa67fe688..aabe29c4827 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -24,10 +24,9 @@ pub use types::{ Uint256, VariableList, Withdrawal, Withdrawals, }; use types::{ - ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadEip6110, - ExecutionPayloadMerge, + ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadEip6110, ExecutionPayloadMerge, + KzgProofs, }; -use types::{ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, KzgProofs}; pub mod auth; pub mod http; @@ -191,7 +190,7 @@ pub struct ExecutionBlockWithTransactions { #[serde(rename = "hash")] pub block_hash: ExecutionBlockHash, pub transactions: Vec, - #[superstruct(only(Capella, Deneb))] + #[superstruct(only(Capella, Deneb, Eip6110))] pub withdrawals: Vec, #[superstruct(only(Eip6110))] pub deposit_receipts: Vec, @@ -264,19 +263,18 @@ impl TryFrom> for ExecutionBlockWithTransactions timestamp: block.timestamp, extra_data: block.extra_data, base_fee_per_gas: block.base_fee_per_gas, + excess_data_gas: block.excess_data_gas, block_hash: block.block_hash, transactions: block - .transactions - .iter() - .map(|tx| Transaction::decode(&Rlp::new(tx))) + .transactions + .iter() + .map(|tx| Transaction::decode(&Rlp::new(tx))) .collect::, _>>()?, withdrawals: Vec::from(block.withdrawals) .into_iter() .map(|withdrawal| withdrawal.into()) .collect(), - excess_data_gas: block.excess_data_gas, - }) - } + }), ExecutionPayload::Eip6110(block) => { Self::Eip6110(ExecutionBlockWithTransactionsEip6110 { parent_hash: block.parent_hash, @@ -308,6 +306,7 @@ impl TryFrom> for ExecutionBlockWithTransactions .collect(), }) } + }; Ok(json_payload) } } @@ -419,7 +418,7 @@ pub struct GetPayloadResponse { #[superstruct(only(Eip6110), partial_getter(rename = "execution_payload_eip6110"))] pub execution_payload: ExecutionPayloadEip6110, pub block_value: Uint256, - #[superstruct(only(Deneb))] + #[superstruct(only(Deneb, Eip6110))] pub blobs_bundle: BlobsBundleV1, } @@ -462,6 +461,7 @@ impl From> GetPayloadResponse::Eip6110(inner) => ( ExecutionPayload::Eip6110(inner.execution_payload), inner.block_value, + Some(inner.blobs_bundle), ), } } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 8c99a37ccaa..d3b37a96871 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -834,23 +834,6 @@ impl HttpJsonRpc { Ok(response.into()) } - pub async fn new_payload_v6110( - &self, - execution_payload: ExecutionPayload, - ) -> Result { - let params = json!([JsonExecutionPayload::from(execution_payload)]); - - let response: JsonPayloadStatusV1 = self - .rpc_request( - ENGINE_NEW_PAYLOAD_V6110, - params, - ENGINE_NEW_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, - ) - .await?; - - Ok(response.into()) - } - pub async fn get_payload_v1( &self, payload_id: PayloadId, @@ -946,7 +929,16 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V3(response).into()) } - ForkName::Eip6110 => self.get_payload_v6110(payload_id).await, + ForkName::Eip6110 => { + let response: JsonGetPayloadResponseV6110 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V6110, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + Ok(JsonGetPayloadResponse::V6110(response).into()) + } ForkName::Base | ForkName::Altair => Err(Error::UnsupportedForkVariant(format!( "called get_payload_v3 with {}", fork_name @@ -956,35 +948,57 @@ impl HttpJsonRpc { pub async fn get_payload_v6110( &self, + fork_name: ForkName, payload_id: PayloadId, ) -> Result, Error> { let params = json!([JsonPayloadIdRequest::from(payload_id)]); - let response: JsonGetPayloadResponseV6110 = self - .rpc_request( - ENGINE_GET_PAYLOAD_V6110, - params, - ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, - ) - .await?; - - Ok(JsonGetPayloadResponse::V6110(response).into()) - } - - pub async fn get_blobs_bundle_v1( - &self, - payload_id: PayloadId, - ) -> Result, Error> { - let params = json!([JsonPayloadIdRequest::from(payload_id)]); - - let response: JsonBlobsBundle = self - .rpc_request( - ENGINE_GET_BLOBS_BUNDLE_V1, - ENGINE_GET_BLOBS_BUNDLE_TIMEOUT, - ) - .await?; - - Ok(response) + match fork_name { + ForkName::Merge => { + let response: JsonGetPayloadResponseV1 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V2, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + Ok(JsonGetPayloadResponse::V1(response).into()) + } + ForkName::Capella => { + let response: JsonGetPayloadResponseV2 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V2, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + Ok(JsonGetPayloadResponse::V2(response).into()) + } + ForkName::Deneb => { + let response: JsonGetPayloadResponseV3 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V3, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + Ok(JsonGetPayloadResponse::V3(response).into()) + } + ForkName::Eip6110 => { + let response: JsonGetPayloadResponseV6110 = self + .rpc_request( + ENGINE_GET_PAYLOAD_V6110, + params, + ENGINE_GET_PAYLOAD_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + Ok(JsonGetPayloadResponse::V6110(response).into()) + } + ForkName::Base | ForkName::Altair => Err(Error::UnsupportedForkVariant(format!( + "called get_payload_v6110 with {}", + fork_name + ))), + } } pub async fn forkchoice_updated_v1( @@ -1012,6 +1026,8 @@ impl HttpJsonRpc { &self, forkchoice_state: ForkchoiceState, payload_attributes: Option, + ) -> Result { + let params = json!([ JsonForkchoiceStateV1::from(forkchoice_state), payload_attributes.map(JsonPayloadAttributes::from) ]); diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index d3cd4ffb205..2753bd4d269 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -7,9 +7,9 @@ use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; use types::{ - Blobs, DepositReceipt, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, - ExecutionPayloadDeneb, ExecutionPayloadEip6110, ExecutionPayloadMerge, FixedVector, - Transaction, Transactions, Unsigned, VariableList, Withdrawal, + EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, + ExecutionPayloadEip6110, ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, + VariableList, Withdrawal, }; #[derive(Debug, PartialEq, Serialize, Deserialize)] @@ -98,8 +98,7 @@ pub struct JsonExecutionPayload { pub base_fee_per_gas: Uint256, pub block_hash: ExecutionBlockHash, #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] - pub transactions: - VariableList, T::MaxTransactionsPerPayload>, + pub transactions: Transactions, #[superstruct(only(V2, V3, V6110))] pub withdrawals: VariableList, #[superstruct(only(V6110))] @@ -367,7 +366,7 @@ pub struct JsonGetPayloadResponse { pub execution_payload: JsonExecutionPayloadV6110, #[serde(with = "eth2_serde_utils::u256_hex_be")] pub block_value: Uint256, - #[superstruct(only(V3))] + #[superstruct(only(V3, V6110))] pub blobs_bundle: JsonBlobsBundleV1, } @@ -397,6 +396,7 @@ impl From> for GetPayloadResponse { GetPayloadResponse::Eip6110(GetPayloadResponseEip6110 { execution_payload: response.execution_payload.into(), block_value: response.block_value, + blobs_bundle: response.blobs_bundle.into(), }) } } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 9e16b7566a8..804f3586493 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -45,17 +45,11 @@ use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; use types::consts::deneb::BLOB_TX_TYPE; use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlobTransaction}; -use types::Withdrawals; -use types::{ - blobs_sidecar::{Blobs, KzgCommitments}, - BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, - ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadEip6110, - ExecutionPayloadMerge, ForkName, -}; use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; use types::{ BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionBlockHash, ExecutionPayload, - ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, ForkName, + ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadEip6110, ExecutionPayloadMerge, + ForkName, }; use types::{KzgProofs, Withdrawals}; use types::{ @@ -263,6 +257,7 @@ impl> BlockProposalContents ExecutionLayer { } }; - let blob_fut = async { - match current_fork { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { - None - } - ForkName::Deneb | ForkName::Eip6110 => { - debug!( - self.log(), - "Issuing engine_getBlobsBundle"; - "suggested_fee_recipient" => ?payload_attributes.suggested_fee_recipient(), - "prev_randao" => ?payload_attributes.prev_randao(), - "timestamp" => payload_attributes.timestamp(), - "parent_hash" => ?parent_hash, - ); - Some(engine.api.get_blobs_bundle_v1::(payload_id).await) - } - } - }; - let payload_fut = async { + let payload_response = async { debug!( self.log(), "Issuing engine_getPayload"; diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index d848a8fa461..e225c483b14 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -19,9 +19,9 @@ use tree_hash_derive::TreeHash; use types::consts::deneb::BLOB_TX_TYPE; use types::transaction::{BlobTransaction, EcdsaSignature, SignedBlobTransaction}; use types::{ - EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, + Blob, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadEip6110, ExecutionPayloadMerge, ForkName, Hash256, - Uint256, + Transaction, Transactions, Uint256, }; const GAS_LIMIT: u64 = 16384; @@ -611,7 +611,7 @@ impl ExecutionBlockGenerator { match execution_payload.fork_name() { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => {} - ForkName::Deneb => { + ForkName::Deneb | ForkName::Eip6110 => { // get random number between 0 and Max Blobs let num_blobs = rand::random::() % T::max_blobs_per_block(); let (bundle, transactions) = self.generate_random_blobs(num_blobs)?; @@ -797,6 +797,7 @@ mod test { None, None, None, + None, ); for i in 0..=TERMINAL_BLOCK { diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 7f4922fecad..992fd104ebd 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -362,6 +362,12 @@ pub async fn handle_rpc( serde_json::to_value(JsonGetPayloadResponseV6110 { execution_payload, block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), + blobs_bundle: maybe_blobs + .ok_or(( + "No blobs returned despite V6110 Payload".to_string(), + GENERIC_ERROR_CODE, + ))? + .into(), }) .unwrap() } @@ -385,6 +391,12 @@ pub async fn handle_rpc( serde_json::to_value(JsonGetPayloadResponseV3 { execution_payload, block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), + blobs_bundle: maybe_blobs + .ok_or(( + "No blobs returned despite V3 Payload".to_string(), + GENERIC_ERROR_CODE, + ))? + .into(), }) .unwrap() } @@ -392,6 +404,12 @@ pub async fn handle_rpc( serde_json::to_value(JsonGetPayloadResponseV6110 { execution_payload, block_value: DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI.into(), + blobs_bundle: maybe_blobs + .ok_or(( + "No blobs returned despite V6110 Payload".to_string(), + GENERIC_ERROR_CODE, + ))? + .into(), }) .unwrap() } diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 1238bc4d050..e9fa3cdb7cc 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -442,13 +442,11 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { let json_payload = serde_json::to_string(&payload).map_err(convert_err)?; let mut message = match fork { - ForkName::Capella | ForkName::Eip4844 | ForkName::Eip6110 => { - BuilderBid::Capella(BuilderBidCapella { - header: serde_json::from_str(json_payload.as_str()).map_err(convert_err)?, - value: to_ssz_rs(&Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI))?, - public_key: self.builder_sk.public_key(), - }) - } + ForkName::Capella => BuilderBid::Capella(BuilderBidCapella { + header: serde_json::from_str(json_payload.as_str()).map_err(convert_err)?, + value: to_ssz_rs(&Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI))?, + public_key: self.builder_sk.public_key(), + }), ForkName::Merge => BuilderBid::Bellatrix(BuilderBidBellatrix { header: serde_json::from_str(json_payload.as_str()).map_err(convert_err)?, value: to_ssz_rs(&Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI))?, diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 023f8a2b8ef..571e466ba41 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -44,7 +44,6 @@ impl MockExecutionLayer { executor: TaskExecutor, terminal_block: u64, shanghai_time: Option, -<<<<<<< HEAD deneb_time: Option, eip6110_time: Option, builder_threshold: Option, @@ -64,7 +63,6 @@ impl MockExecutionLayer { spec.terminal_block_hash, shanghai_time, deneb_time, - deneb_time, eip6110_time, kzg, ); diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index e6b0a24e76d..3b25a7fd091 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -102,6 +102,7 @@ impl MockServer { None, // FIXME(capella): should this be the default? None, // FIXME(deneb): should this be the default? None, // FIXME(eip6110): should this be the default? + None, // FIXME(deneb): should this be the default? ) } @@ -129,6 +130,7 @@ impl MockServer { shanghai_time, deneb_time, eip6110_time, + kzg, ); let ctx: Arc> = Arc::new(Context { @@ -189,8 +191,8 @@ impl MockServer { terminal_block_hash: ExecutionBlockHash, shanghai_time: Option, deneb_time: Option, - kzg: Option, eip6110_time: Option, + kzg: Option, ) -> Self { Self::new_with_config( handle, diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index d6c5ada0071..69f3c282d50 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -14,7 +14,7 @@ pub fn build_block_contents { Ok(BlockContents::Block(block)) } - ForkName::Deneb => { + ForkName::Deneb | ForkName::Eip6110 => { let block_root = &block.canonical_root(); if let Some(blob_sidecars) = chain.proposal_blob_cache.pop(block_root) { let block_and_blobs = BeaconBlockAndBlobSidecars { diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index d722cf6c9b7..a58dd54931a 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -68,7 +68,7 @@ pub async fn publish_block( crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; block.into() } - SignedBeaconBlock::Deneb(_) => { + SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Eip6110(_) => { crate::publish_pubsub_message(network_tx, PubsubMessage::BeaconBlock(block.clone()))?; if let Some(signed_blobs) = maybe_blobs { for (blob_index, blob) in signed_blobs.clone().into_iter().enumerate() { diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index 7fbfadd5544..f4e8f486a7a 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -18,8 +18,8 @@ use tokio_util::codec::{Decoder, Encoder}; use types::{light_client_bootstrap::LightClientBootstrap, BlobSidecar}; use types::{ EthSpec, ForkContext, ForkName, Hash256, SignedBeaconBlock, SignedBeaconBlockAltair, - SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockEip6110, - SignedBeaconBlockMerge, + SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, + SignedBeaconBlockEip6110, SignedBeaconBlockMerge, }; use unsigned_varint::codec::Uvi; diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index eb99e6b91ea..b9b11385a7b 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -12,8 +12,9 @@ use types::{ Attestation, AttesterSlashing, EthSpec, ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, - SignedBeaconBlockDeneb, SignedBeaconBlockMerge, SignedBlobSidecar, SignedBlsToExecutionChange, - SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, + SignedBeaconBlockDeneb, SignedBeaconBlockEip6110, SignedBeaconBlockMerge, SignedBlobSidecar, + SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, + SyncCommitteeMessage, SyncSubnetId, }; #[derive(Debug, Clone, PartialEq)] @@ -186,15 +187,15 @@ impl PubsubMessage { ), Some(ForkName::Capella) => SignedBeaconBlock::::Capella( SignedBeaconBlockCapella::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, + .map_err(|e| format!("{:?}", e))?, ), Some(ForkName::Deneb) => SignedBeaconBlock::::Deneb( SignedBeaconBlockDeneb::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, + .map_err(|e| format!("{:?}", e))?, ), Some(ForkName::Eip6110) => SignedBeaconBlock::::Eip6110( SignedBeaconBlockEip6110::from_ssz_bytes(data) - .map_err(|e| format!("{:?}", e))?, + .map_err(|e| format!("{:?}", e))?, ), None => { return Err(format!( diff --git a/beacon_node/store/src/impls/execution_payload.rs b/beacon_node/store/src/impls/execution_payload.rs index c34fb30c7d2..29c435aaa95 100644 --- a/beacon_node/store/src/impls/execution_payload.rs +++ b/beacon_node/store/src/impls/execution_payload.rs @@ -1,7 +1,7 @@ use crate::{DBColumn, Error, StoreItem}; use ssz::{Decode, Encode}; use types::{ - BlobsSidecar, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, + BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadEip6110, ExecutionPayloadMerge, }; diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 581ddadd084..810c3ae0683 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1316,7 +1316,7 @@ impl> ForkVersionDeserialize D, >(value, fork_name)?)) } - ForkName::Deneb => Ok(BlockContents::BlockAndBlobSidecars( + ForkName::Deneb | ForkName::Eip6110 => Ok(BlockContents::BlockAndBlobSidecars( BeaconBlockAndBlobSidecars::deserialize_by_fork::<'de, D>(value, fork_name)?, )), } @@ -1380,6 +1380,7 @@ impl> From SignedBlockContents::Block(block), //TODO: error handling, this should be try from SignedBeaconBlock::Deneb(_block) => todo!(), + SignedBeaconBlock::Eip6110(_block) => todo!(), } } } diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index adcc3a05be0..83fe7861674 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -111,8 +111,8 @@ pub fn initialize_beacon_state_from_eth1( // Override latest execution payload header. // See https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/beacon-chain.md#testing - if let Some(ExecutionPayloadHeader::Deneb(header)) = execution_payload_header { - *state.latest_execution_payload_header_deneb_mut()? = header; + if let Some(ExecutionPayloadHeader::Deneb(ref header)) = execution_payload_header { + *state.latest_execution_payload_header_deneb_mut()? = header.clone(); } } diff --git a/consensus/state_processing/src/upgrade/eip6110.rs b/consensus/state_processing/src/upgrade/eip6110.rs index 0afe5f0e118..68b25209fc3 100644 --- a/consensus/state_processing/src/upgrade/eip6110.rs +++ b/consensus/state_processing/src/upgrade/eip6110.rs @@ -7,7 +7,7 @@ pub fn upgrade_to_eip6110( spec: &ChainSpec, ) -> Result<(), Error> { let epoch = pre_state.current_epoch(); - let pre = pre_state.as_eip4844_mut()?; + let pre = pre_state.as_deneb_mut()?; let previous_fork_version = pre.fork.current_version; diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index e40da335c71..68ab10aceed 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -129,8 +129,8 @@ impl> BeaconBlock { /// on the fork slot. pub fn any_from_ssz_bytes(bytes: &[u8]) -> Result { BeaconBlockEip6110::from_ssz_bytes(bytes) - .map(BeaconBlock::Eip6110) - .or_else(|_| BeaconBlockDeneb::from_ssz_bytes(bytes).map(BeaconBlock::Deneb)) + .map(BeaconBlock::Eip6110) + .or_else(|_| BeaconBlockDeneb::from_ssz_bytes(bytes).map(BeaconBlock::Deneb)) .or_else(|_| BeaconBlockCapella::from_ssz_bytes(bytes).map(BeaconBlock::Capella)) .or_else(|_| BeaconBlockMerge::from_ssz_bytes(bytes).map(BeaconBlock::Merge)) .or_else(|_| BeaconBlockAltair::from_ssz_bytes(bytes).map(BeaconBlock::Altair)) @@ -226,6 +226,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockRef<'a, T, Payl BeaconBlockRef::Merge { .. } => ForkName::Merge, BeaconBlockRef::Capella { .. } => ForkName::Capella, BeaconBlockRef::Deneb { .. } => ForkName::Deneb, + BeaconBlockRef::Eip6110 { .. } => ForkName::Eip6110, } } @@ -1000,7 +1001,7 @@ mod tests { // It's invalid to have an Eip4844 block with a epoch lower than the fork epoch. let bad_block = { let mut bad = good_block.clone(); - *bad.slot_mut() = eip4844_slot; + *bad.slot_mut() = deneb_slot; bad }; diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index eade993bd56..293e33a4ca8 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -1,5 +1,6 @@ -use crate::test_utils::TestRandom; +use crate::payload::BlindedPayloadDeneb; use crate::*; +use crate::{payload::FullPayloadDeneb, test_utils::TestRandom}; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -73,7 +74,7 @@ pub struct BeaconBlockBody = FullPay #[superstruct(only(Capella, Deneb, Eip6110))] pub bls_to_execution_changes: VariableList, - #[superstruct(only(Eip4844, Eip6110))] + #[superstruct(only(Deneb, Eip6110))] pub blob_kzg_commitments: KzgCommitments, #[superstruct(only(Base, Altair))] #[ssz(skip_serializing, skip_deserializing)] diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 09d7dc75c9a..53f68fdf001 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -297,7 +297,7 @@ where only(Deneb), partial_getter(rename = "latest_execution_payload_header_deneb") )] - pub latest_execution_payload_header: ExecutionPayloadHeaderEipDeneb, + pub latest_execution_payload_header: ExecutionPayloadHeaderDeneb, #[superstruct( only(Eip6110), partial_getter(rename = "latest_execution_payload_header_eip6110") @@ -445,6 +445,7 @@ impl BeaconState { BeaconState::Merge { .. } => ForkName::Merge, BeaconState::Capella { .. } => ForkName::Capella, BeaconState::Deneb { .. } => ForkName::Deneb, + BeaconState::Eip6110 { .. } => ForkName::Eip6110, } } @@ -1969,6 +1970,9 @@ pub mod ssz_tagged_beacon_state { body, )?)), ForkName::Deneb => Ok(BeaconState::Deneb(BeaconStateDeneb::from_ssz_bytes(body)?)), + ForkName::Eip6110 => Ok(BeaconState::Eip6110(BeaconStateEip6110::from_ssz_bytes( + body, + )?)), } } } diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 1b59888f443..0fd5982b279 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -100,9 +100,7 @@ impl ExecutionPayloadHeader { ForkName::Capella => { ExecutionPayloadHeaderCapella::from_ssz_bytes(bytes).map(Self::Capella) } - ForkName::Deneb => { - ExecutionPayloadHeaderDeneb::from_ssz_bytes(bytes).map(Self::Deneb) - } + ForkName::Deneb => ExecutionPayloadHeaderDeneb::from_ssz_bytes(bytes).map(Self::Deneb), ForkName::Eip6110 => { ExecutionPayloadHeaderEip6110::from_ssz_bytes(bytes).map(Self::Eip6110) } @@ -165,7 +163,7 @@ impl ExecutionPayloadHeaderCapella { } } -impl ExecutionPayloadHeaderEip4844 { +impl ExecutionPayloadHeaderDeneb { pub fn upgrade_to_eip6110(&self) -> ExecutionPayloadHeaderEip6110 { ExecutionPayloadHeaderEip6110 { parent_hash: self.parent_hash, diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index 15cffe39254..b573fc37d04 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -96,7 +96,7 @@ impl ForkName { ForkName::Merge => Some(ForkName::Altair), ForkName::Capella => Some(ForkName::Merge), ForkName::Deneb => Some(ForkName::Capella), - ForkName::Eip6110 => Some(ForkName::Eip4844), + ForkName::Eip6110 => Some(ForkName::Deneb), } } diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index e27452196b1..74762eed8a1 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -110,7 +110,7 @@ pub use crate::attestation_data::AttestationData; pub use crate::attestation_duty::AttestationDuty; pub use crate::attester_slashing::AttesterSlashing; pub use crate::beacon_block::{ - BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockEipDeneb, + BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockDeneb, BeaconBlockEip6110, BeaconBlockMerge, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, EmptyBlock, }; @@ -163,9 +163,9 @@ pub use crate::light_client_optimistic_update::LightClientOptimisticUpdate; pub use crate::participation_flags::ParticipationFlags; pub use crate::participation_list::ParticipationList; pub use crate::payload::{ - AbstractExecPayload, BlindedPayload, BlindedPayloadCapella, BlindedPayloadEipDeneb, + AbstractExecPayload, BlindedPayload, BlindedPayloadCapella, BlindedPayloadDeneb, BlindedPayloadEip6110, BlindedPayloadMerge, BlindedPayloadRef, BlockType, ExecPayload, - FullPayload, FullPayloadCapella, FullPayloadEipDeneb, FullPayloadEip6110, FullPayloadMerge, + FullPayload, FullPayloadCapella, FullPayloadDeneb, FullPayloadEip6110, FullPayloadMerge, FullPayloadRef, OwnedExecPayload, }; pub use crate::pending_attestation::PendingAttestation; @@ -178,8 +178,9 @@ pub use crate::shuffling_id::AttestationShufflingId; pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof; pub use crate::signed_beacon_block::{ ssz_tagged_signed_beacon_block, SignedBeaconBlock, SignedBeaconBlockAltair, - SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockEip6110, SignedBeaconBlockHash, - SignedBeaconBlockMerge, SignedBlindedBeaconBlock, + SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, + SignedBeaconBlockEip6110, SignedBeaconBlockHash, SignedBeaconBlockMerge, + SignedBlindedBeaconBlock, }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_blob::*; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index d7287389014..1f2ba3600eb 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -89,7 +89,7 @@ pub trait AbstractExecPayload: + Copy + From<&'a Self::Merge> + From<&'a Self::Capella> - + From<&'a Self::Deneb>; + + From<&'a Self::Deneb> + From<&'a Self::Eip6110>; type Merge: OwnedExecPayload @@ -272,7 +272,7 @@ impl ExecPayload for FullPayload { match self { FullPayload::Merge(_) => Err(Error::IncorrectStateVariant), FullPayload::Capella(_) => Err(Error::IncorrectStateVariant), - FullPayload::Eip4844(_) => Err(Error::IncorrectStateVariant), + FullPayload::Deneb(_) => Err(Error::IncorrectStateVariant), FullPayload::Eip6110(ref inner) => Ok(inner.execution_payload.deposit_receipts.clone()), } } @@ -393,7 +393,7 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { match self { FullPayloadRef::Merge(_) => Err(Error::IncorrectStateVariant), FullPayloadRef::Capella(_) => Err(Error::IncorrectStateVariant), - FullPayloadRef::Eip4844(_) => Err(Error::IncorrectStateVariant), + FullPayloadRef::Deneb(_) => Err(Error::IncorrectStateVariant), FullPayloadRef::Eip6110(inner) => Ok(inner.execution_payload.deposit_receipts.clone()), } } @@ -568,9 +568,7 @@ impl ExecPayload for BlindedPayload { BlindedPayload::Capella(ref inner) => { Ok(inner.execution_payload_header.withdrawals_root) } - BlindedPayload::Deneb(ref inner) => { - Ok(inner.execution_payload_header.withdrawals_root) - } + BlindedPayload::Deneb(ref inner) => Ok(inner.execution_payload_header.withdrawals_root), BlindedPayload::Eip6110(ref inner) => { Ok(inner.execution_payload_header.withdrawals_root) } @@ -672,9 +670,7 @@ impl<'b, T: EthSpec> ExecPayload for BlindedPayloadRef<'b, T> { BlindedPayloadRef::Capella(inner) => { Ok(inner.execution_payload_header.withdrawals_root) } - BlindedPayloadRef::Deneb(inner) => { - Ok(inner.execution_payload_header.withdrawals_root) - } + BlindedPayloadRef::Deneb(inner) => Ok(inner.execution_payload_header.withdrawals_root), BlindedPayloadRef::Eip6110(inner) => { Ok(inner.execution_payload_header.withdrawals_root) } diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index 2e3ba670c89..3b45403a0c2 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -669,6 +669,9 @@ pub mod ssz_tagged_signed_beacon_block { ForkName::Deneb => Ok(SignedBeaconBlock::Deneb( SignedBeaconBlockDeneb::from_ssz_bytes(body)?, )), + ForkName::Eip6110 => Ok(SignedBeaconBlock::Eip6110( + SignedBeaconBlockEip6110::from_ssz_bytes(body)?, + )), } } } diff --git a/lcli/src/parse_ssz.rs b/lcli/src/parse_ssz.rs index b78dcf2a76c..191bafecd59 100644 --- a/lcli/src/parse_ssz.rs +++ b/lcli/src/parse_ssz.rs @@ -65,8 +65,8 @@ pub fn run_parse_ssz(matches: &ArgMatches) -> Result<(), String> { "state_merge" => decode_and_print::>(&bytes, format)?, "state_capella" => decode_and_print::>(&bytes, format)?, "state_deneb" => decode_and_print::>(&bytes, format)?, - "blobs_sidecar" => decode_and_print::>(&bytes, format)?, "state_eip6110" => decode_and_print::>(&bytes, format)?, + "blob_sidecar" => decode_and_print::>(&bytes, format)?, other => return Err(format!("Unknown type: {}", other)), }; diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index 2a008412dbb..77092c69c0e 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -381,7 +381,8 @@ impl Operation for DepositReceipt { && fork_name != ForkName::Altair && fork_name != ForkName::Merge && fork_name != ForkName::Capella - && fork_name != ForkName::Eip4844 + && fork_name != ForkName::Deneb + && fork_name != ForkName::Eip6110 } fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { diff --git a/testing/ef_tests/src/cases/transition.rs b/testing/ef_tests/src/cases/transition.rs index e5ab8ad10f9..b0f2d74d501 100644 --- a/testing/ef_tests/src/cases/transition.rs +++ b/testing/ef_tests/src/cases/transition.rs @@ -57,7 +57,7 @@ impl LoadCase for TransitionTest { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec.capella_fork_epoch = Some(Epoch::new(0)); - spec.eip4844_fork_epoch = Some(Epoch::new(0)); + spec.deneb_fork_epoch = Some(Epoch::new(0)); spec.eip6110_fork_epoch = Some(metadata.fork_epoch); } }