Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 59a9c1a

Browse files
adlerjohnliamsi
authored and
evan-forbes
committedSep 22, 2021
Implement spec-compliant share splitting (#246)
* Export block data compute shares. * Refactor to use ShareSize constant directly. * Change message splitting to prefix namespace ID. * Implement chunking for contiguous. * Add termination condition. * Rename append contiguous to split contiguous. * Update test for small tx. * Add test for two contiguous. * Make tx and msg adjusted share sizes exported constants. * Panic on hopefully-unreachable condition instead of silently skipping. * Update hardcoded response for block format. Co-authored-by: Ismail Khoffi <[email protected]>
1 parent f9df60e commit 59a9c1a

File tree

4 files changed

+142
-44
lines changed

4 files changed

+142
-44
lines changed
 

‎types/block.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -1120,26 +1120,26 @@ type IntermediateStateRoots struct {
11201120
RawRootsList []tmbytes.HexBytes `json:"intermediate_roots"`
11211121
}
11221122

1123-
func (roots IntermediateStateRoots) splitIntoShares(shareSize int) NamespacedShares {
1123+
func (roots IntermediateStateRoots) splitIntoShares() NamespacedShares {
11241124
shares := make([]NamespacedShare, 0)
11251125
for _, root := range roots.RawRootsList {
11261126
rawData, err := root.MarshalDelimited()
11271127
if err != nil {
11281128
panic(fmt.Sprintf("app returned intermediate state root that can not be encoded %#v", root))
11291129
}
1130-
shares = appendToShares(shares, consts.IntermediateStateRootsNamespaceID, rawData, shareSize)
1130+
shares = appendToShares(shares, consts.IntermediateStateRootsNamespaceID, rawData)
11311131
}
11321132
return shares
11331133
}
11341134

1135-
func (msgs Messages) splitIntoShares(shareSize int) NamespacedShares {
1135+
func (msgs Messages) splitIntoShares() NamespacedShares {
11361136
shares := make([]NamespacedShare, 0)
11371137
for _, m := range msgs.MessagesList {
11381138
rawData, err := m.MarshalDelimited()
11391139
if err != nil {
11401140
panic(fmt.Sprintf("app accepted a Message that can not be encoded %#v", m))
11411141
}
1142-
shares = appendToShares(shares, m.NamespaceID, rawData, shareSize)
1142+
shares = appendToShares(shares, m.NamespaceID, rawData)
11431143
}
11441144
return shares
11451145
}
@@ -1346,7 +1346,7 @@ func (data *EvidenceData) FromProto(eviData *tmproto.EvidenceList) error {
13461346
return nil
13471347
}
13481348

1349-
func (data *EvidenceData) splitIntoShares(shareSize int) NamespacedShares {
1349+
func (data *EvidenceData) splitIntoShares() NamespacedShares {
13501350
shares := make([]NamespacedShare, 0)
13511351
for _, ev := range data.Evidence {
13521352
var rawData []byte
@@ -1367,7 +1367,7 @@ func (data *EvidenceData) splitIntoShares(shareSize int) NamespacedShares {
13671367
if err != nil {
13681368
panic(fmt.Sprintf("evidence included in evidence pool that can not be encoded %#v, err: %v", ev, err))
13691369
}
1370-
shares = appendToShares(shares, consts.EvidenceNamespaceID, rawData, shareSize)
1370+
shares = appendToShares(shares, consts.EvidenceNamespaceID, rawData)
13711371
}
13721372
return shares
13731373
}

‎types/shares.go

+69-11
Original file line numberDiff line numberDiff line change
@@ -59,35 +59,93 @@ func (m Message) MarshalDelimited() ([]byte, error) {
5959
return append(lenBuf[:n], m.Data...), nil
6060
}
6161

62-
func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte, shareSize int) []NamespacedShare {
63-
if len(rawData) < shareSize {
64-
rawShare := rawData
65-
paddedShare := zeroPadIfNecessary(rawShare, shareSize)
62+
// appendToShares appends raw data as shares.
63+
// Used for messages.
64+
func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare {
65+
if len(rawData) < consts.MsgShareSize {
66+
rawShare := []byte(append(nid, rawData...))
67+
paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize)
68+
share := NamespacedShare{paddedShare, nid}
69+
shares = append(shares, share)
70+
} else { // len(rawData) >= MsgShareSize
71+
shares = append(shares, split(rawData, nid)...)
72+
}
73+
return shares
74+
}
75+
76+
// splitContiguous splits multiple raw data contiguously as shares.
77+
// Used for transactions, intermediate state roots, and evidence.
78+
func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
79+
shares := make([]NamespacedShare, 0)
80+
// Index into the outer slice of rawDatas
81+
outerIndex := 0
82+
// Index into the inner slice of rawDatas
83+
innerIndex := 0
84+
for outerIndex < len(rawDatas) {
85+
var rawData []byte
86+
startIndex := 0
87+
rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, consts.TxShareSize)
88+
rawShare := []byte(append(append(nid, byte(startIndex)), rawData...))
89+
paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize)
6690
share := NamespacedShare{paddedShare, nid}
6791
shares = append(shares, share)
68-
} else { // len(rawData) >= shareSize
69-
shares = append(shares, split(rawData, shareSize, nid)...)
7092
}
7193
return shares
7294
}
7395

7496
// TODO(ismail): implement corresponding merge method for clients requesting
7597
// shares for a particular namespace
76-
func split(rawData []byte, shareSize int, nid namespace.ID) []NamespacedShare {
98+
func split(rawData []byte, nid namespace.ID) []NamespacedShare {
7799
shares := make([]NamespacedShare, 0)
78-
firstRawShare := rawData[:shareSize]
100+
firstRawShare := []byte(append(nid, rawData[:consts.MsgShareSize]...))
79101
shares = append(shares, NamespacedShare{firstRawShare, nid})
80-
rawData = rawData[shareSize:]
102+
rawData = rawData[consts.MsgShareSize:]
81103
for len(rawData) > 0 {
82-
shareSizeOrLen := min(shareSize, len(rawData))
83-
paddedShare := zeroPadIfNecessary(rawData[:shareSizeOrLen], shareSize)
104+
shareSizeOrLen := min(consts.MsgShareSize, len(rawData))
105+
rawShare := []byte(append(nid, rawData[:shareSizeOrLen]...))
106+
paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize)
84107
share := NamespacedShare{paddedShare, nid}
85108
shares = append(shares, share)
86109
rawData = rawData[shareSizeOrLen:]
87110
}
88111
return shares
89112
}
90113

114+
// getNextChunk gets the next chunk for contiguous shares
115+
// Precondition: none of the slices in rawDatas is zero-length
116+
// This precondition should always hold at this point since zero-length txs are simply invalid.
117+
func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) {
118+
rawData := make([]byte, 0, width)
119+
startIndex := 0
120+
firstBytesToFetch := 0
121+
122+
curIndex := 0
123+
for curIndex < width && outerIndex < len(rawDatas) {
124+
bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex, width-curIndex)
125+
if bytesToFetch == 0 {
126+
panic("zero-length contiguous share data is invalid")
127+
}
128+
if curIndex == 0 {
129+
firstBytesToFetch = bytesToFetch
130+
}
131+
// If we've already placed some data in this chunk, that means
132+
// a new data segment begins
133+
if curIndex != 0 {
134+
// Offset by the fixed reserved bytes at the beginning of the share
135+
startIndex = firstBytesToFetch + consts.NamespaceSize + consts.ShareReservedBytes
136+
}
137+
rawData = append(rawData, rawDatas[outerIndex][innerIndex:innerIndex+bytesToFetch]...)
138+
innerIndex += bytesToFetch
139+
if innerIndex >= len(rawDatas[outerIndex]) {
140+
innerIndex = 0
141+
outerIndex++
142+
}
143+
curIndex += bytesToFetch
144+
}
145+
146+
return rawData, outerIndex, innerIndex, startIndex
147+
}
148+
91149
func GenerateTailPaddingShares(n int, shareWidth int) NamespacedShares {
92150
shares := make([]NamespacedShare, n)
93151
for i := 0; i < n; i++ {

‎types/shares_test.go

+65-25
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,12 @@ import (
1111
)
1212

1313
type splitter interface {
14-
splitIntoShares(shareSize int) NamespacedShares
14+
splitIntoShares() NamespacedShares
1515
}
1616

1717
func TestMakeShares(t *testing.T) {
1818
reservedTxNamespaceID := append(bytes.Repeat([]byte{0}, 7), 1)
1919
reservedEvidenceNamespaceID := append(bytes.Repeat([]byte{0}, 7), 3)
20-
// resveredIntermediateStateRootsNamespaceID := append(bytes.Repeat([]byte{0}, 7), 2)
2120
val := NewMockPV()
2221
blockID := makeBlockID([]byte("blockhash"), 1000, []byte("partshash"))
2322
blockID2 := makeBlockID([]byte("blockhash2"), 1000, []byte("partshash"))
@@ -38,12 +37,11 @@ func TestMakeShares(t *testing.T) {
3837
}
3938
msg1Marshaled, _ := msg1.MarshalDelimited()
4039
if err != nil {
41-
t.Fatalf("Could not encode evidence: %v, error: %v", testEvidence, err)
40+
t.Fatalf("Could not encode evidence: %v, error: %v\n", testEvidence, err)
4241
}
4342

4443
type args struct {
45-
data splitter
46-
shareSize int
44+
data splitter
4745
}
4846
tests := []struct {
4947
name string
@@ -55,59 +53,101 @@ func TestMakeShares(t *testing.T) {
5553
data: &EvidenceData{
5654
Evidence: []Evidence{testEvidence},
5755
},
58-
shareSize: consts.ShareSize,
5956
}, NamespacedShares{NamespacedShare{
60-
Share: testEvidenceBytes[:consts.ShareSize],
61-
ID: reservedEvidenceNamespaceID,
57+
Share: append(
58+
append(reservedEvidenceNamespaceID, byte(0)),
59+
testEvidenceBytes[:consts.TxShareSize]...,
60+
),
61+
ID: reservedEvidenceNamespaceID,
6262
}, NamespacedShare{
63-
Share: zeroPadIfNecessary(testEvidenceBytes[consts.ShareSize:], consts.ShareSize),
64-
ID: reservedEvidenceNamespaceID,
63+
Share: append(
64+
append(reservedEvidenceNamespaceID, byte(0)),
65+
zeroPadIfNecessary(testEvidenceBytes[consts.TxShareSize:], consts.TxShareSize)...,
66+
),
67+
ID: reservedEvidenceNamespaceID,
6568
}},
6669
},
6770
{"small LL Tx",
6871
args{
69-
data: Txs{smolTx},
70-
shareSize: consts.ShareSize,
72+
data: Txs{smolTx},
7173
},
7274
NamespacedShares{
7375
NamespacedShare{
74-
Share: zeroPadIfNecessary(smolTxLenDelimited, consts.ShareSize),
75-
ID: reservedTxNamespaceID,
76+
Share: append(
77+
append(reservedTxNamespaceID, byte(0)),
78+
zeroPadIfNecessary(smolTxLenDelimited, consts.TxShareSize)...,
79+
),
80+
ID: reservedTxNamespaceID,
7681
},
7782
},
7883
},
7984
{"one large LL Tx",
8085
args{
81-
data: Txs{largeTx},
82-
shareSize: consts.ShareSize,
86+
data: Txs{largeTx},
8387
},
8488
NamespacedShares{
8589
NamespacedShare{
86-
Share: Share(largeTxLenDelimited[:consts.ShareSize]),
87-
ID: reservedTxNamespaceID,
90+
Share: append(
91+
append(reservedTxNamespaceID, byte(0)),
92+
largeTxLenDelimited[:consts.TxShareSize]...,
93+
),
94+
ID: reservedTxNamespaceID,
8895
},
8996
NamespacedShare{
90-
Share: zeroPadIfNecessary(largeTxLenDelimited[consts.ShareSize:], consts.ShareSize),
91-
ID: reservedTxNamespaceID,
97+
Share: append(
98+
append(reservedTxNamespaceID, byte(0)),
99+
zeroPadIfNecessary(largeTxLenDelimited[consts.TxShareSize:], consts.TxShareSize)...,
100+
),
101+
ID: reservedTxNamespaceID,
102+
},
103+
},
104+
},
105+
{"large then small LL Tx",
106+
args{
107+
data: Txs{largeTx, smolTx},
108+
},
109+
NamespacedShares{
110+
NamespacedShare{
111+
Share: append(
112+
append(reservedTxNamespaceID, byte(0)),
113+
largeTxLenDelimited[:consts.TxShareSize]...,
114+
),
115+
ID: reservedTxNamespaceID,
116+
},
117+
NamespacedShare{
118+
Share: append(
119+
append(reservedTxNamespaceID, byte(len(largeTxLenDelimited)-consts.TxShareSize+consts.NamespaceSize+consts.ShareReservedBytes)),
120+
zeroPadIfNecessary(
121+
append(largeTxLenDelimited[consts.TxShareSize:], smolTxLenDelimited...),
122+
consts.TxShareSize,
123+
)...,
124+
),
125+
ID: reservedTxNamespaceID,
92126
},
93127
},
94128
},
95129
{"ll-app message",
96130
args{
97-
data: Messages{[]Message{msg1}},
98-
shareSize: consts.ShareSize,
131+
data: Messages{[]Message{msg1}},
99132
},
100133
NamespacedShares{
101-
NamespacedShare{zeroPadIfNecessary(msg1Marshaled, consts.ShareSize), msg1.NamespaceID},
134+
NamespacedShare{
135+
Share: append(
136+
[]byte(msg1.NamespaceID),
137+
zeroPadIfNecessary(msg1Marshaled, consts.MsgShareSize)...,
138+
),
139+
ID: msg1.NamespaceID,
140+
},
102141
},
103142
},
104143
}
105144
for i, tt := range tests {
106145
tt := tt // stupid scopelint :-/
107146
i := i
108147
t.Run(tt.name, func(t *testing.T) {
109-
if got := tt.args.data.splitIntoShares(tt.args.shareSize); !reflect.DeepEqual(got, tt.want) {
110-
t.Errorf("%v: makeShares() = \n%v\nwant\n%v", i, got, tt.want)
148+
got := tt.args.data.splitIntoShares()
149+
if !reflect.DeepEqual(got, tt.want) {
150+
t.Errorf("%v: makeShares() = \n%+v\nwant\n%+v\n", i, got, tt.want)
111151
}
112152
})
113153
}

‎types/tx.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -80,14 +80,14 @@ func (txs Txs) Proof(i int) TxProof {
8080
}
8181
}
8282

83-
func (txs Txs) splitIntoShares(shareSize int) NamespacedShares {
83+
func (txs Txs) splitIntoShares() NamespacedShares {
8484
shares := make([]NamespacedShare, 0)
8585
for _, tx := range txs {
8686
rawData, err := tx.MarshalDelimited()
8787
if err != nil {
8888
panic(fmt.Sprintf("included Tx in mem-pool that can not be encoded %v", tx))
8989
}
90-
shares = appendToShares(shares, consts.TxNamespaceID, rawData, shareSize)
90+
shares = appendToShares(shares, consts.TxNamespaceID, rawData)
9191
}
9292
return shares
9393
}

0 commit comments

Comments
 (0)
Please sign in to comment.