Derivation of type information/reflection/metadata #2184
Description
What features would you like from zerocopy?
I'm looking at zerocopy as one of the potential serialization methods in blockchain environment.
Another one will likely be https://github.com/paritytech/parity-scale-codec, which is not as low-level and will be a bit more expensive to work with than casting memory bytes into structs.
What I'm looking for is a way to have some sort of type information or reflection about data structures zerocopy works with, such that external users can understand what blob of bytes contains and display it in human-readable way, auto-generate UI, etc.
A good example of this is https://github.com/paritytech/scale-info for mentioned parity-scale-codec.
What scale-info allows to do is to generate something like this (used in Substrate-based blockchains to dynamically auto-generate user interfaces):
Details
types: [
{
id: 0
type: {
path: [
sp_core
crypto
AccountId32
]
params: []
def: {
Composite: {
fields: [
{
name: null
type: 1
typeName: [u8; 32]
docs: []
}
]
}
}
docs: []
}
}
{
id: 1
type: {
path: []
params: []
def: {
Array: {
len: 32
type: 2
}
}
docs: []
}
}
{
id: 2
type: {
path: []
params: []
def: {
Primitive: U8
}
docs: []
}
}
{
id: 3
type: {
path: [
frame_system
AccountInfo
]
params: [
{
name: Nonce
type: 4
}
{
name: AccountData
type: 5
}
]
def: {
Composite: {
fields: [
{
name: nonce
type: 4
typeName: Nonce
docs: []
}
{
name: consumers
type: 4
typeName: RefCount
docs: []
}
{
name: providers
type: 4
typeName: RefCount
docs: []
}
{
name: sufficients
type: 4
typeName: RefCount
docs: []
}
{
name: data
type: 5
typeName: AccountData
docs: []
}
]
}
}
docs: []
}
}
{
id: 4
type: {
path: []
params: []
def: {
Primitive: U32
}
docs: []
}
}
{
id: 5
type: {
path: [
pallet_balances
types
AccountData
]
params: [
{
name: Balance
type: 6
}
]
def: {
Composite: {
fields: [
{
name: free
type: 6
typeName: Balance
docs: []
}
{
name: reserved
type: 6
typeName: Balance
docs: []
}
{
name: frozen
type: 6
typeName: Balance
docs: []
}
{
name: flags
type: 7
typeName: ExtraFlags
docs: []
}
]
}
}
docs: []
}
}
{
id: 6
type: {
path: []
params: []
def: {
Primitive: U128
}
docs: []
}
}
{
id: 7
type: {
path: [
pallet_balances
types
ExtraFlags
]
params: []
def: {
Composite: {
fields: [
{
name: null
type: 6
typeName: u128
docs: []
}
]
}
}
docs: []
}
}
{
id: 8
type: {
path: []
params: []
def: {
Primitive: Bool
}
docs: []
}
}
{
id: 9
type: {
path: [
frame_support
dispatch
PerDispatchClass
]
params: [
{
name: T
type: 10
}
]
def: {
Composite: {
fields: [
{
name: normal
type: 10
typeName: T
docs: []
}
{
name: operational
type: 10
typeName: T
docs: []
}
{
name: mandatory
type: 10
typeName: T
docs: []
}
]
}
}
docs: []
}
}
{
id: 10
type: {
path: [
sp_weights
weight_v2
Weight
]
params: []
def: {
Composite: {
fields: [
{
name: ref_time
type: 11
typeName: u64
docs: []
}
{
name: proof_size
type: 11
typeName: u64
docs: []
}
]
}
}
docs: []
}
}
{
id: 11
type: {
path: []
params: []
def: {
Compact: {
type: 12
}
}
docs: []
}
}
{
id: 12
type: {
path: []
params: []
def: {
Primitive: U64
}
docs: []
}
}
{
id: 13
type: {
path: [
primitive_types
H256
]
params: []
def: {
Composite: {
fields: [
{
name: null
type: 1
typeName: [u8; 32]
docs: []
}
]
}
}
docs: []
}
}
{
id: 14
type: {
path: []
params: []
def: {
Sequence: {
type: 2
}
}
docs: []
}
}
{
id: 15
type: {
path: [
sp_runtime
generic
digest
Digest
]
params: []
def: {
Composite: {
fields: [
{
name: logs
type: 16
typeName: Vec<DigestItem>
docs: []
}
]
}
}
docs: []
}
}
{
id: 16
type: {
path: []
params: []
def: {
Sequence: {
type: 17
}
}
docs: []
}
}
{
id: 17
type: {
path: [
sp_runtime
generic
digest
DigestItem
]
params: []
def: {
Variant: {
variants: [
{
name: PreRuntime
fields: [
{
name: null
type: 18
typeName: ConsensusEngineId
docs: []
}
{
name: null
type: 14
typeName: Vec<u8>
docs: []
}
]
index: 6
docs: []
}
{
name: Consensus
fields: [
{
name: null
type: 18
typeName: ConsensusEngineId
docs: []
}
{
name: null
type: 14
typeName: Vec<u8>
docs: []
}
]
index: 4
docs: []
}
{
name: Seal
fields: [
{
name: null
type: 18
typeName: ConsensusEngineId
docs: []
}
{
name: null
type: 14
typeName: Vec<u8>
docs: []
}
]
index: 5
docs: []
}
{
name: Other
fields: [
{
name: null
type: 14
typeName: Vec<u8>
docs: []
}
]
index: 0
docs: []
}
{
name: RuntimeEnvironmentUpdated
fields: []
index: 8
docs: []
}
]
}
}
docs: []
}
}
{
id: 18
type: {
path: []
params: []
def: {
Array: {
len: 4
type: 2
}
}
docs: []
}
}
{
id: 19
type: {
path: []
params: []
def: {
Sequence: {
type: 20
}
}
docs: []
}
}
{
id: 20
type: {
path: [
frame_system
EventRecord
]
params: [
{
name: E
type: 21
}
{
name: T
type: 13
}
]
def: {
Composite: {
fields: [
{
name: phase
type: 497
typeName: Phase
docs: []
}
{
name: event
type: 21
typeName: E
docs: []
}
{
name: topics
type: 101
typeName: Vec<T>
docs: []
}
]
}
}
docs: []
}
}
{
id: 21
type: {
path: [
polkadot_runtime
RuntimeEvent
]
params: []
def: {
Variant: {
variants: [
{
name: System
fields: [
{
name: null
type: 22
typeName: frame_system::Event<Runtime>
docs: []
}
]
index: 0
docs: []
}
{
name: Scheduler
fields: [
{
name: null
type: 31
typeName: pallet_scheduler::Event<Runtime>
docs: []
}
]
index: 1
docs: []
}
{
name: Preimage
fields: [
{
name: null
type: 36
typeName: pallet_preimage::Event<Runtime>
docs: []
}
]
index: 10
docs: []
}
{
name: Indices
fields: [
{
name: null
type: 37
typeName: pallet_indices::Event<Runtime>
docs: []
}
]
index: 4
docs: []
}
{
name: Balances
fields: [
{
name: null
type: 38
typeName: pallet_balances::Event<Runtime>
docs: []
}
]
index: 5
docs: []
}
{
name: TransactionPayment
fields: [
{
name: null
type: 40
typeName: pallet_transaction_payment::Event<Runtime>
docs: []
}
]
index: 32
docs: []
}
{
name: Staking
fields: [
{
name: null
type: 41
typeName: pallet_staking::Event<Runtime>
docs: []
}
]
index: 7
docs: []
}
{
name: Offences
fields: [
{
name: null
type: 47
typeName: pallet_offences::Event
docs: []
}
]
index: 8
docs: []
}
{
name: Session
fields: [
{
name: null
type: 49
typeName: pallet_session::Event
docs: []
}
]
index: 9
docs: []
}
{
name: Grandpa
fields: [
{
name: null
type: 50
typeName: pallet_grandpa::Event
docs: []
}
]
index: 11
docs: []
}
{
name: Treasury
fields: [
{
name: null
type: 54
typeName: pallet_treasury::Event<Runtime>
docs: []
}
]
index: 19
docs: []
}
{
name: ConvictionVoting
fields: [
{
name: null
type: 89
typeName: pallet_conviction_voting::Event<Runtime>
docs: []
}
]
index: 20
docs: []
}
{
name: Referenda
fields: [
{
name: null
type: 90
typeName: pallet_referenda::Event<Runtime>
docs: []
}
]
index: 21
docs: []
}
{
name: Whitelist
fields: [
{
name: null
type: 449
typeName: pallet_whitelist::Event<Runtime>
docs: []
}
]
index: 23
docs: []
}
{
name: Parameters
fields: [
{
name: null
type: 454
typeName: pallet_parameters::Event<Runtime>
docs: []
}
]
index: 27
docs: []
}
{
name: Claims
fields: [
{
name: null
type: 460
typeName: claims::Event<Runtime>
docs: []
}
]
index: 24
docs: []
}
{
name: Vesting
fields: [
{
name: null
type: 461
typeName: pallet_vesting::Event<Runtime>
docs: []
}
]
index: 25
docs: []
}
{
name: Utility
fields: [
{
name: null
type: 462
typeName: pallet_utility::Event
docs: []
}
]
index: 26
docs: []
}
{
name: Proxy
fields: [
{
name: null
type: 463
typeName: pallet_proxy::Event<Runtime>
docs: []
}
]
index: 29
docs: []
}
{
name: Multisig
fields: [
{
name: null
type: 464
typeName: pallet_multisig::Event<Runtime>
docs: []
}
]
index: 30
docs: []
}
{
name: Bounties
fields: [
{
name: null
type: 465
typeName: pallet_bounties::Event<Runtime>
docs: []
}
]
index: 34
docs: []
}
{
name: ChildBounties
fields: [
{
name: null
type: 466
typeName: pallet_child_bounties::Event<Runtime>
docs: []
}
]
index: 38
docs: []
}
{
name: ElectionProviderMultiPhase
fields: [
{
name: null
type: 467
typeName: pallet_election_provider_multi_phase::Event<Runtime>
docs: []
}
]
index: 36
docs: []
}
{
name: VoterList
fields: [
{
name: null
type: 471
typeName: pallet_bags_list::Event<Runtime, pallet_bags_list::Instance1>
docs: []
}
]
index: 37
docs: []
}
{
name: NominationPools
fields: [
{
name: null
type: 472
typeName: pallet_nomination_pools::Event<Runtime>
docs: []
}
]
index: 39
docs: []
}
{
name: FastUnstake
fields: [
{
name: null
type: 473
typeName: pallet_fast_unstake::Event<Runtime>
docs: []
}
]
index: 40
docs: []
}
{
name: ParaInclusion
fields: [
{
name: null
type: 474
typeName: parachains_inclusion::Event<Runtime>
docs: []
}
]
index: 53
docs: []
}
{
name: Paras
fields: [
{
name: null
type: 478
typeName: parachains_paras::Event
docs: []
}
]
index: 56
docs: []
}
{
name: Hrmp
fields: [
{
name: null
type: 479
typeName: parachains_hrmp::Event<Runtime>
docs: []
}
]
index: 60
docs: []
}
{
name: ParasDisputes
fields: [
{
name: null
type: 480
typeName: parachains_disputes::Event<Runtime>
docs: []
}
]
index: 62
docs: []
}
{
name: OnDemand
fields: [
{
name: null
type: 483
typeName: parachains_assigner_on_demand::Event<Runtime>
docs: []
}
]
index: 64
docs: []
}
{
name: Registrar
fields: [
{
name: null
type: 484
typeName: paras_registrar::Event<Runtime>
docs: []
}
]
index: 70
docs: []
}
{
name: Slots
fields: [
{
name: null
type: 485
typeName: slots::Event<Runtime>
docs: []
}
]
index: 71
docs: []
}
{
name: Auctions
fields: [
{
name: null
type: 486
typeName: auctions::Event<Runtime>
docs: []
}
]
index: 72
docs: []
}
{
name: Crowdloan
fields: [
{
name: null
type: 487
typeName: crowdloan::Event<Runtime>
docs: []
}
]
index: 73
docs: []
}
{
name: Coretime
fields: [
{
name: null
type: 488
typeName: coretime::Event<Runtime>
docs: []
}
]
index: 74
docs: []
}
{
name: StateTrieMigration
fields: [
{
name: null
type: 489
typeName: pallet_state_trie_migration::Event<Runtime>
docs: []
}
]
index: 98
docs: []
}
{
name: XcmPallet
fields: [
{
name: null
type: 492
typeName: pallet_xcm::Event<Runtime>
docs: []
}
]
index: 99
docs: []
}
{
name: MessageQueue
fields: [
{
name: null
type: 494
typeName: pallet_message_queue::Event<Runtime>
docs: []
}
]
index: 100
docs: []
}
{
name: AssetRate
fields: [
{
name: null
type: 496
typeName: pallet_asset_rate::Event<Runtime>
docs: []
}
]
index: 101
docs: []
}
]
}
}
docs: []
}
}
There is a collection of crates that allow to do various manipulations with such type information: https://github.com/paritytech?q=scale&language=rust
Example of UI that can be rendered with such metadata:
Details
It would contain information of the type and all of the fields/variants it contains recursively and exhaustively all the way down to the types from Rust standard library. Implementation for standard library would be a part of zerocopy and the rest derivable my the user on custom data structures. Information about alignment, padding and potentially endianness of the target environment would be included as well.
I was thinking that generating a C data structure might be useful for this purpose, but it will likely be harder to parse than JSON or similar tree-like format and I have not seen derives quite like that either.