Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 1cb4030

Browse files
evan-forbesevan-forbes
committedApr 5, 2021
review feedback: split up share merging and splitting into their own files
1 parent 76d7a4b commit 1cb4030

File tree

3 files changed

+470
-465
lines changed

3 files changed

+470
-465
lines changed
 

‎types/share_merging.go

+333
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
package types
2+
3+
import (
4+
"bytes"
5+
"encoding/binary"
6+
"errors"
7+
8+
"github.com/gogo/protobuf/proto"
9+
tmbytes "github.com/lazyledger/lazyledger-core/libs/bytes"
10+
tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types"
11+
"github.com/lazyledger/rsmt2d"
12+
)
13+
14+
// DataFromSquare extracts block data from an extended data square.
15+
func DataFromSquare(eds *rsmt2d.ExtendedDataSquare) (Data, error) {
16+
originalWidth := eds.Width() / 2
17+
18+
// sort block data by namespace
19+
// define a slice for the raw share data of each type
20+
var (
21+
txsShares [][]byte
22+
isrShares [][]byte
23+
evdShares [][]byte
24+
msgShares [][]byte
25+
)
26+
27+
// iterate over each row index
28+
for x := uint(0); x < originalWidth; x++ {
29+
// iterate over each col index
30+
for y := uint(0); y < originalWidth; y++ {
31+
// sort the data of that share types via namespace
32+
share := eds.Cell(x, y)
33+
nid := share[:NamespaceSize]
34+
switch {
35+
case bytes.Equal(TxNamespaceID, nid):
36+
txsShares = append(txsShares, share)
37+
38+
case bytes.Equal(IntermediateStateRootsNamespaceID, nid):
39+
isrShares = append(isrShares, share)
40+
41+
case bytes.Equal(EvidenceNamespaceID, nid):
42+
evdShares = append(evdShares, share)
43+
44+
case bytes.Equal(TailPaddingNamespaceID, nid):
45+
continue
46+
47+
// ignore unused but reserved namespaces
48+
case bytes.Compare(nid, MaxReservedNamespace) < 1:
49+
continue
50+
51+
// every other namespaceID should be a message
52+
default:
53+
msgShares = append(msgShares, share)
54+
}
55+
}
56+
}
57+
58+
// pass the raw share data to their respective parsers
59+
txs, err := parseTxs(txsShares)
60+
if err != nil {
61+
return Data{}, err
62+
}
63+
64+
isrs, err := parseISRs(isrShares)
65+
if err != nil {
66+
return Data{}, err
67+
}
68+
69+
evd, err := parseEvd(evdShares)
70+
if err != nil {
71+
return Data{}, err
72+
}
73+
74+
msgs, err := parseMsgs(msgShares)
75+
if err != nil {
76+
return Data{}, err
77+
}
78+
79+
return Data{
80+
Txs: txs,
81+
IntermediateStateRoots: isrs,
82+
Evidence: evd,
83+
Messages: msgs,
84+
}, nil
85+
}
86+
87+
// parseTxs collects all of the transactions from the shares provided
88+
func parseTxs(shares [][]byte) (Txs, error) {
89+
// parse the sharse
90+
rawTxs, err := processContiguousShares(shares)
91+
if err != nil {
92+
return nil, err
93+
}
94+
95+
// convert to the Tx type
96+
txs := make(Txs, len(rawTxs))
97+
for i := 0; i < len(txs); i++ {
98+
txs[i] = Tx(rawTxs[i])
99+
}
100+
101+
return txs, nil
102+
}
103+
104+
// parseISRs collects all the intermediate state roots from the shares provided
105+
func parseISRs(shares [][]byte) (IntermediateStateRoots, error) {
106+
rawISRs, err := processContiguousShares(shares)
107+
if err != nil {
108+
return IntermediateStateRoots{}, err
109+
}
110+
111+
ISRs := make([]tmbytes.HexBytes, len(rawISRs))
112+
for i := 0; i < len(ISRs); i++ {
113+
ISRs[i] = rawISRs[i]
114+
}
115+
116+
return IntermediateStateRoots{RawRootsList: ISRs}, nil
117+
}
118+
119+
// parseEvd collects all evidence from the shares provided.
120+
func parseEvd(shares [][]byte) (EvidenceData, error) {
121+
// the raw data returned does not have length delimiters or namespaces and
122+
// is ready to be unmarshaled
123+
rawEvd, err := processContiguousShares(shares)
124+
if err != nil {
125+
return EvidenceData{}, err
126+
}
127+
128+
evdList := make(EvidenceList, len(rawEvd))
129+
130+
// parse into protobuf bytes
131+
for i := 0; i < len(rawEvd); i++ {
132+
// unmarshal the evidence
133+
var protoEvd tmproto.Evidence
134+
err := proto.Unmarshal(rawEvd[i], &protoEvd)
135+
if err != nil {
136+
return EvidenceData{}, err
137+
}
138+
evd, err := EvidenceFromProto(&protoEvd)
139+
if err != nil {
140+
return EvidenceData{}, err
141+
}
142+
143+
evdList[i] = evd
144+
}
145+
146+
return EvidenceData{Evidence: evdList}, nil
147+
}
148+
149+
// parseMsgs collects all messages from the shares provided
150+
func parseMsgs(shares [][]byte) (Messages, error) {
151+
msgList, err := parseMsgShares(shares)
152+
if err != nil {
153+
return MessagesEmpty, err
154+
}
155+
156+
return Messages{
157+
MessagesList: msgList,
158+
}, nil
159+
}
160+
161+
// processContiguousShares takes raw shares and extracts out transactions,
162+
// intermediate state roots, or evidence. The returned [][]byte do have
163+
// namespaces or length delimiters and are ready to be unmarshalled
164+
func processContiguousShares(shares [][]byte) (txs [][]byte, err error) {
165+
if len(shares) == 0 {
166+
return nil, nil
167+
}
168+
169+
ss := newShareStack(shares)
170+
return ss.resolve()
171+
}
172+
173+
// shareStack hold variables for peel
174+
type shareStack struct {
175+
shares [][]byte
176+
txLen uint64
177+
txs [][]byte
178+
cursor int
179+
}
180+
181+
func newShareStack(shares [][]byte) *shareStack {
182+
return &shareStack{shares: shares}
183+
}
184+
185+
func (ss *shareStack) resolve() ([][]byte, error) {
186+
if len(ss.shares) == 0 {
187+
return nil, nil
188+
}
189+
err := ss.peel(ss.shares[0][NamespaceSize+ShareReservedBytes:], true)
190+
return ss.txs, err
191+
}
192+
193+
// peel recursively parses each chunk of data (either a transaction,
194+
// intermediate state root, or evidence) and adds it to the underlying slice of data.
195+
func (ss *shareStack) peel(share []byte, delimited bool) (err error) {
196+
if delimited {
197+
var txLen uint64
198+
share, txLen, err = parseDelimiter(share)
199+
if err != nil {
200+
return err
201+
}
202+
if txLen == 0 {
203+
return nil
204+
}
205+
ss.txLen = txLen
206+
}
207+
// safeLen describes the point in the share where it can be safely split. If
208+
// split beyond this point, it is possible to break apart a length
209+
// delimiter, which will result in incorrect share merging
210+
safeLen := len(share) - binary.MaxVarintLen64
211+
if safeLen < 0 {
212+
safeLen = 0
213+
}
214+
if ss.txLen <= uint64(safeLen) {
215+
ss.txs = append(ss.txs, share[:ss.txLen])
216+
share = share[ss.txLen:]
217+
return ss.peel(share, true)
218+
}
219+
// add the next share to the current share to continue merging if possible
220+
if len(ss.shares) > ss.cursor+1 {
221+
ss.cursor++
222+
share := append(share, ss.shares[ss.cursor][NamespaceSize+ShareReservedBytes:]...)
223+
return ss.peel(share, false)
224+
}
225+
// collect any remaining data
226+
if ss.txLen <= uint64(len(share)) {
227+
ss.txs = append(ss.txs, share[:ss.txLen])
228+
share = share[ss.txLen:]
229+
return ss.peel(share, true)
230+
}
231+
return errors.New("failure to parse block data: transaction length exceeded data length")
232+
}
233+
234+
// parseMsgShares iterates through raw shares and separates the contiguous chunks
235+
// of data. It is only used for Messages, i.e. shares with a non-reserved namespace.
236+
func parseMsgShares(shares [][]byte) ([]Message, error) {
237+
if len(shares) == 0 {
238+
return nil, nil
239+
}
240+
241+
// set the first nid and current share
242+
nid := shares[0][:NamespaceSize]
243+
currentShare := shares[0][NamespaceSize:]
244+
245+
// find and remove the msg len delimiter
246+
currentShare, msgLen, err := parseDelimiter(currentShare)
247+
if err != nil {
248+
return nil, err
249+
}
250+
251+
var msgs []Message
252+
for cursor := uint64(0); cursor < uint64(len(shares)); {
253+
var msg Message
254+
currentShare, nid, cursor, msgLen, msg, err = nextMsg(
255+
shares,
256+
currentShare,
257+
nid,
258+
cursor,
259+
msgLen,
260+
)
261+
if err != nil {
262+
return nil, err
263+
}
264+
if msg.Data != nil {
265+
msgs = append(msgs, msg)
266+
}
267+
}
268+
269+
return msgs, nil
270+
}
271+
272+
func nextMsg(
273+
shares [][]byte,
274+
current,
275+
nid []byte,
276+
cursor,
277+
msgLen uint64,
278+
) ([]byte, []byte, uint64, uint64, Message, error) {
279+
switch {
280+
// the message uses all of the current share data and at least some of the
281+
// next share
282+
case msgLen > uint64(len(current)):
283+
// add the next share to the current one and try again
284+
cursor++
285+
current = append(current, shares[cursor][NamespaceSize:]...)
286+
return nextMsg(shares, current, nid, cursor, msgLen)
287+
288+
// the msg we're looking for is contained in the current share
289+
case msgLen <= uint64(len(current)):
290+
msg := Message{nid, current[:msgLen]}
291+
cursor++
292+
293+
// call it a day if the work is done
294+
if cursor >= uint64(len(shares)) {
295+
return nil, nil, cursor, 0, msg, nil
296+
}
297+
298+
nextNid := shares[cursor][:NamespaceSize]
299+
next, msgLen, err := parseDelimiter(shares[cursor][NamespaceSize:])
300+
return next, nextNid, cursor, msgLen, msg, err
301+
}
302+
// this code is unreachable but the compiler doesn't know that
303+
return nil, nil, 0, 0, MessageEmpty, nil
304+
}
305+
306+
// parseDelimiter finds and returns the length delimiter of the message provided
307+
// while also removing the delimiter bytes from the input
308+
func parseDelimiter(input []byte) ([]byte, uint64, error) {
309+
if len(input) == 0 {
310+
return input, 0, nil
311+
}
312+
313+
l := binary.MaxVarintLen64
314+
if len(input) < binary.MaxVarintLen64 {
315+
l = len(input)
316+
}
317+
318+
delimiter := zeroPadIfNecessary(input[:l], binary.MaxVarintLen64)
319+
320+
// read the length of the message
321+
r := bytes.NewBuffer(delimiter)
322+
msgLen, err := binary.ReadUvarint(r)
323+
if err != nil {
324+
return nil, 0, err
325+
}
326+
327+
// calculate the number of bytes used by the delimiter
328+
lenBuf := make([]byte, binary.MaxVarintLen64)
329+
n := binary.PutUvarint(lenBuf, msgLen)
330+
331+
// return the input without the length delimiter
332+
return input[n:], msgLen, nil
333+
}

‎types/share_splitting.go

+137
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
package types
2+
3+
import (
4+
"bytes"
5+
6+
"github.com/lazyledger/nmt/namespace"
7+
)
8+
9+
// appendToShares appends raw data as shares.
10+
// Used for messages.
11+
func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare {
12+
if len(rawData) <= MsgShareSize {
13+
rawShare := append(append(
14+
make([]byte, 0, len(nid)+len(rawData)),
15+
nid...),
16+
rawData...,
17+
)
18+
paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
19+
share := NamespacedShare{paddedShare, nid}
20+
shares = append(shares, share)
21+
} else { // len(rawData) > MsgShareSize
22+
shares = append(shares, splitMessage(rawData, nid)...)
23+
}
24+
return shares
25+
}
26+
27+
// splitMessage breaks the data in a message into the minimum number of
28+
// namespaced shares
29+
func splitMessage(rawData []byte, nid namespace.ID) []NamespacedShare {
30+
shares := make([]NamespacedShare, 0)
31+
firstRawShare := append(append(
32+
make([]byte, 0, ShareSize),
33+
nid...),
34+
rawData[:MsgShareSize]...,
35+
)
36+
shares = append(shares, NamespacedShare{firstRawShare, nid})
37+
rawData = rawData[MsgShareSize:]
38+
for len(rawData) > 0 {
39+
shareSizeOrLen := min(MsgShareSize, len(rawData))
40+
rawShare := append(append(
41+
make([]byte, 0, ShareSize),
42+
nid...),
43+
rawData[:shareSizeOrLen]...,
44+
)
45+
paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
46+
share := NamespacedShare{paddedShare, nid}
47+
shares = append(shares, share)
48+
rawData = rawData[shareSizeOrLen:]
49+
}
50+
return shares
51+
}
52+
53+
// splitContiguous splits multiple raw data contiguously as shares.
54+
// Used for transactions, intermediate state roots, and evidence.
55+
func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
56+
shares := make([]NamespacedShare, 0)
57+
// Index into the outer slice of rawDatas
58+
outerIndex := 0
59+
// Index into the inner slice of rawDatas
60+
innerIndex := 0
61+
for outerIndex < len(rawDatas) {
62+
var rawData []byte
63+
startIndex := 0
64+
rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, TxShareSize)
65+
rawShare := append(append(append(
66+
make([]byte, 0, len(nid)+1+len(rawData)),
67+
nid...),
68+
byte(startIndex)),
69+
rawData...)
70+
paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
71+
share := NamespacedShare{paddedShare, nid}
72+
shares = append(shares, share)
73+
}
74+
return shares
75+
}
76+
77+
// getNextChunk gets the next chunk for contiguous shares
78+
// Precondition: none of the slices in rawDatas is zero-length
79+
// This precondition should always hold at this point since zero-length txs are simply invalid.
80+
func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) {
81+
rawData := make([]byte, 0, width)
82+
startIndex := 0
83+
firstBytesToFetch := 0
84+
85+
curIndex := 0
86+
for curIndex < width && outerIndex < len(rawDatas) {
87+
bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex, width-curIndex)
88+
if bytesToFetch == 0 {
89+
panic("zero-length contiguous share data is invalid")
90+
}
91+
if curIndex == 0 {
92+
firstBytesToFetch = bytesToFetch
93+
}
94+
// If we've already placed some data in this chunk, that means
95+
// a new data segment begins
96+
if curIndex != 0 {
97+
// Offset by the fixed reserved bytes at the beginning of the share
98+
startIndex = firstBytesToFetch + NamespaceSize + ShareReservedBytes
99+
}
100+
rawData = append(rawData, rawDatas[outerIndex][innerIndex:innerIndex+bytesToFetch]...)
101+
innerIndex += bytesToFetch
102+
if innerIndex >= len(rawDatas[outerIndex]) {
103+
innerIndex = 0
104+
outerIndex++
105+
}
106+
curIndex += bytesToFetch
107+
}
108+
109+
return rawData, outerIndex, innerIndex, startIndex
110+
}
111+
112+
func GenerateTailPaddingShares(n int, shareWidth int) NamespacedShares {
113+
shares := make([]NamespacedShare, n)
114+
for i := 0; i < n; i++ {
115+
shares[i] = NamespacedShare{bytes.Repeat([]byte{0}, shareWidth), TailPaddingNamespaceID}
116+
}
117+
return shares
118+
}
119+
120+
func min(a, b int) int {
121+
if a <= b {
122+
return a
123+
}
124+
return b
125+
}
126+
127+
func zeroPadIfNecessary(share []byte, width int) []byte {
128+
oldLen := len(share)
129+
if oldLen < width {
130+
missingBytes := width - oldLen
131+
padByte := []byte{0}
132+
padding := bytes.Repeat(padByte, missingBytes)
133+
share = append(share, padding...)
134+
return share
135+
}
136+
return share
137+
}

‎types/shares.go

-465
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,9 @@
11
package types
22

33
import (
4-
"bytes"
54
"encoding/binary"
6-
"errors"
75

8-
"github.com/gogo/protobuf/proto"
9-
tmbytes "github.com/lazyledger/lazyledger-core/libs/bytes"
10-
tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types"
116
"github.com/lazyledger/nmt/namespace"
12-
"github.com/lazyledger/rsmt2d"
137
)
148

159
// Share contains the raw share data without the corresponding namespace.
@@ -58,462 +52,3 @@ func (m Message) MarshalDelimited() ([]byte, error) {
5852
n := binary.PutUvarint(lenBuf, length)
5953
return append(lenBuf[:n], m.Data...), nil
6054
}
61-
62-
// /////////////////////////////
63-
// Splitting
64-
// ////////////////////////////
65-
66-
// appendToShares appends raw data as shares.
67-
// Used for messages.
68-
func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare {
69-
if len(rawData) <= MsgShareSize {
70-
rawShare := append(append(
71-
make([]byte, 0, len(nid)+len(rawData)),
72-
nid...),
73-
rawData...,
74-
)
75-
paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
76-
share := NamespacedShare{paddedShare, nid}
77-
shares = append(shares, share)
78-
} else { // len(rawData) > MsgShareSize
79-
shares = append(shares, splitMessage(rawData, nid)...)
80-
}
81-
return shares
82-
}
83-
84-
// splitMessage breaks the data in a message into the minimum number of
85-
// namespaced shares
86-
func splitMessage(rawData []byte, nid namespace.ID) []NamespacedShare {
87-
shares := make([]NamespacedShare, 0)
88-
firstRawShare := append(append(
89-
make([]byte, 0, ShareSize),
90-
nid...),
91-
rawData[:MsgShareSize]...,
92-
)
93-
shares = append(shares, NamespacedShare{firstRawShare, nid})
94-
rawData = rawData[MsgShareSize:]
95-
for len(rawData) > 0 {
96-
shareSizeOrLen := min(MsgShareSize, len(rawData))
97-
rawShare := append(append(
98-
make([]byte, 0, ShareSize),
99-
nid...),
100-
rawData[:shareSizeOrLen]...,
101-
)
102-
paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
103-
share := NamespacedShare{paddedShare, nid}
104-
shares = append(shares, share)
105-
rawData = rawData[shareSizeOrLen:]
106-
}
107-
return shares
108-
}
109-
110-
// splitContiguous splits multiple raw data contiguously as shares.
111-
// Used for transactions, intermediate state roots, and evidence.
112-
func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
113-
shares := make([]NamespacedShare, 0)
114-
// Index into the outer slice of rawDatas
115-
outerIndex := 0
116-
// Index into the inner slice of rawDatas
117-
innerIndex := 0
118-
for outerIndex < len(rawDatas) {
119-
var rawData []byte
120-
startIndex := 0
121-
rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, TxShareSize)
122-
rawShare := append(append(append(
123-
make([]byte, 0, len(nid)+1+len(rawData)),
124-
nid...),
125-
byte(startIndex)),
126-
rawData...)
127-
paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
128-
share := NamespacedShare{paddedShare, nid}
129-
shares = append(shares, share)
130-
}
131-
return shares
132-
}
133-
134-
// getNextChunk gets the next chunk for contiguous shares
135-
// Precondition: none of the slices in rawDatas is zero-length
136-
// This precondition should always hold at this point since zero-length txs are simply invalid.
137-
func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) {
138-
rawData := make([]byte, 0, width)
139-
startIndex := 0
140-
firstBytesToFetch := 0
141-
142-
curIndex := 0
143-
for curIndex < width && outerIndex < len(rawDatas) {
144-
bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex, width-curIndex)
145-
if bytesToFetch == 0 {
146-
panic("zero-length contiguous share data is invalid")
147-
}
148-
if curIndex == 0 {
149-
firstBytesToFetch = bytesToFetch
150-
}
151-
// If we've already placed some data in this chunk, that means
152-
// a new data segment begins
153-
if curIndex != 0 {
154-
// Offset by the fixed reserved bytes at the beginning of the share
155-
startIndex = firstBytesToFetch + NamespaceSize + ShareReservedBytes
156-
}
157-
rawData = append(rawData, rawDatas[outerIndex][innerIndex:innerIndex+bytesToFetch]...)
158-
innerIndex += bytesToFetch
159-
if innerIndex >= len(rawDatas[outerIndex]) {
160-
innerIndex = 0
161-
outerIndex++
162-
}
163-
curIndex += bytesToFetch
164-
}
165-
166-
return rawData, outerIndex, innerIndex, startIndex
167-
}
168-
169-
func GenerateTailPaddingShares(n int, shareWidth int) NamespacedShares {
170-
shares := make([]NamespacedShare, n)
171-
for i := 0; i < n; i++ {
172-
shares[i] = NamespacedShare{bytes.Repeat([]byte{0}, shareWidth), TailPaddingNamespaceID}
173-
}
174-
return shares
175-
}
176-
177-
func min(a, b int) int {
178-
if a <= b {
179-
return a
180-
}
181-
return b
182-
}
183-
184-
func zeroPadIfNecessary(share []byte, width int) []byte {
185-
oldLen := len(share)
186-
if oldLen < width {
187-
missingBytes := width - oldLen
188-
padByte := []byte{0}
189-
padding := bytes.Repeat(padByte, missingBytes)
190-
share = append(share, padding...)
191-
return share
192-
}
193-
return share
194-
}
195-
196-
// /////////////////////////////
197-
// Merging
198-
// ////////////////////////////
199-
200-
// DataFromSquare extracts block data from an extended data square.
201-
func DataFromSquare(eds *rsmt2d.ExtendedDataSquare) (Data, error) {
202-
originalWidth := eds.Width() / 2
203-
204-
// sort block data by namespace
205-
// define a slice for the raw share data of each type
206-
var (
207-
txsShares [][]byte
208-
isrShares [][]byte
209-
evdShares [][]byte
210-
msgShares [][]byte
211-
)
212-
213-
// iterate over each row index
214-
for x := uint(0); x < originalWidth; x++ {
215-
// iterate over each col index
216-
for y := uint(0); y < originalWidth; y++ {
217-
// sort the data of that share types via namespace
218-
share := eds.Cell(x, y)
219-
nid := share[:NamespaceSize]
220-
switch {
221-
case bytes.Equal(TxNamespaceID, nid):
222-
txsShares = append(txsShares, share)
223-
224-
case bytes.Equal(IntermediateStateRootsNamespaceID, nid):
225-
isrShares = append(isrShares, share)
226-
227-
case bytes.Equal(EvidenceNamespaceID, nid):
228-
evdShares = append(evdShares, share)
229-
230-
case bytes.Equal(TailPaddingNamespaceID, nid):
231-
continue
232-
233-
// ignore unused but reserved namespaces
234-
case bytes.Compare(nid, MaxReservedNamespace) < 1:
235-
continue
236-
237-
// every other namespaceID should be a message
238-
default:
239-
msgShares = append(msgShares, share)
240-
}
241-
}
242-
}
243-
244-
// pass the raw share data to their respective parsers
245-
txs, err := parseTxs(txsShares)
246-
if err != nil {
247-
return Data{}, err
248-
}
249-
250-
isrs, err := parseISRs(isrShares)
251-
if err != nil {
252-
return Data{}, err
253-
}
254-
255-
evd, err := parseEvd(evdShares)
256-
if err != nil {
257-
return Data{}, err
258-
}
259-
260-
msgs, err := parseMsgs(msgShares)
261-
if err != nil {
262-
return Data{}, err
263-
}
264-
265-
return Data{
266-
Txs: txs,
267-
IntermediateStateRoots: isrs,
268-
Evidence: evd,
269-
Messages: msgs,
270-
}, nil
271-
}
272-
273-
// parseTxs collects all of the transactions from the shares provided
274-
func parseTxs(shares [][]byte) (Txs, error) {
275-
// parse the sharse
276-
rawTxs, err := processContiguousShares(shares)
277-
if err != nil {
278-
return nil, err
279-
}
280-
281-
// convert to the Tx type
282-
txs := make(Txs, len(rawTxs))
283-
for i := 0; i < len(txs); i++ {
284-
txs[i] = Tx(rawTxs[i])
285-
}
286-
287-
return txs, nil
288-
}
289-
290-
// parseISRs collects all the intermediate state roots from the shares provided
291-
func parseISRs(shares [][]byte) (IntermediateStateRoots, error) {
292-
rawISRs, err := processContiguousShares(shares)
293-
if err != nil {
294-
return IntermediateStateRoots{}, err
295-
}
296-
297-
ISRs := make([]tmbytes.HexBytes, len(rawISRs))
298-
for i := 0; i < len(ISRs); i++ {
299-
ISRs[i] = rawISRs[i]
300-
}
301-
302-
return IntermediateStateRoots{RawRootsList: ISRs}, nil
303-
}
304-
305-
// parseEvd collects all evidence from the shares provided.
306-
func parseEvd(shares [][]byte) (EvidenceData, error) {
307-
// the raw data returned does not have length delimiters or namespaces and
308-
// is ready to be unmarshaled
309-
rawEvd, err := processContiguousShares(shares)
310-
if err != nil {
311-
return EvidenceData{}, err
312-
}
313-
314-
evdList := make(EvidenceList, len(rawEvd))
315-
316-
// parse into protobuf bytes
317-
for i := 0; i < len(rawEvd); i++ {
318-
// unmarshal the evidence
319-
var protoEvd tmproto.Evidence
320-
err := proto.Unmarshal(rawEvd[i], &protoEvd)
321-
if err != nil {
322-
return EvidenceData{}, err
323-
}
324-
evd, err := EvidenceFromProto(&protoEvd)
325-
if err != nil {
326-
return EvidenceData{}, err
327-
}
328-
329-
evdList[i] = evd
330-
}
331-
332-
return EvidenceData{Evidence: evdList}, nil
333-
}
334-
335-
// parseMsgs collects all messages from the shares provided
336-
func parseMsgs(shares [][]byte) (Messages, error) {
337-
msgList, err := parseMsgShares(shares)
338-
if err != nil {
339-
return MessagesEmpty, err
340-
}
341-
342-
return Messages{
343-
MessagesList: msgList,
344-
}, nil
345-
}
346-
347-
// processContiguousShares takes raw shares and extracts out transactions,
348-
// intermediate state roots, or evidence. The returned [][]byte do have
349-
// namespaces or length delimiters and are ready to be unmarshalled
350-
func processContiguousShares(shares [][]byte) (txs [][]byte, err error) {
351-
if len(shares) == 0 {
352-
return nil, nil
353-
}
354-
355-
ss := newShareStack(shares)
356-
return ss.resolve()
357-
}
358-
359-
// shareStack hold variables for peel
360-
type shareStack struct {
361-
shares [][]byte
362-
txLen uint64
363-
txs [][]byte
364-
cursor int
365-
}
366-
367-
func newShareStack(shares [][]byte) *shareStack {
368-
return &shareStack{shares: shares}
369-
}
370-
371-
func (ss *shareStack) resolve() ([][]byte, error) {
372-
if len(ss.shares) == 0 {
373-
return nil, nil
374-
}
375-
err := ss.peel(ss.shares[0][NamespaceSize+ShareReservedBytes:], true)
376-
return ss.txs, err
377-
}
378-
379-
// peel recursively parses each chunk of data (either a transaction,
380-
// intermediate state root, or evidence) and adds it to the underlying slice of data.
381-
func (ss *shareStack) peel(share []byte, delimited bool) (err error) {
382-
if delimited {
383-
var txLen uint64
384-
share, txLen, err = parseDelimiter(share)
385-
if err != nil {
386-
return err
387-
}
388-
if txLen == 0 {
389-
return nil
390-
}
391-
ss.txLen = txLen
392-
}
393-
// safeLen describes the point in the share where it can be safely split. If
394-
// split beyond this point, it is possible to break apart a length
395-
// delimiter, which will result in incorrect share merging
396-
safeLen := len(share) - binary.MaxVarintLen64
397-
if safeLen < 0 {
398-
safeLen = 0
399-
}
400-
if ss.txLen <= uint64(safeLen) {
401-
ss.txs = append(ss.txs, share[:ss.txLen])
402-
share = share[ss.txLen:]
403-
return ss.peel(share, true)
404-
}
405-
// add the next share to the current share to continue merging if possible
406-
if len(ss.shares) > ss.cursor+1 {
407-
ss.cursor++
408-
share := append(share, ss.shares[ss.cursor][NamespaceSize+ShareReservedBytes:]...)
409-
return ss.peel(share, false)
410-
}
411-
// collect any remaining data
412-
if ss.txLen <= uint64(len(share)) {
413-
ss.txs = append(ss.txs, share[:ss.txLen])
414-
share = share[ss.txLen:]
415-
return ss.peel(share, true)
416-
}
417-
return errors.New("failure to parse block data: transaction length exceeded data length")
418-
}
419-
420-
// parseMsgShares iterates through raw shares and separates the contiguous chunks
421-
// of data. It is only used for Messages, i.e. shares with a non-reserved namespace.
422-
func parseMsgShares(shares [][]byte) ([]Message, error) {
423-
if len(shares) == 0 {
424-
return nil, nil
425-
}
426-
427-
// set the first nid and current share
428-
nid := shares[0][:NamespaceSize]
429-
currentShare := shares[0][NamespaceSize:]
430-
431-
// find and remove the msg len delimiter
432-
currentShare, msgLen, err := parseDelimiter(currentShare)
433-
if err != nil {
434-
return nil, err
435-
}
436-
437-
var msgs []Message
438-
for cursor := uint64(0); cursor < uint64(len(shares)); {
439-
var msg Message
440-
currentShare, nid, cursor, msgLen, msg, err = nextMsg(
441-
shares,
442-
currentShare,
443-
nid,
444-
cursor,
445-
msgLen,
446-
)
447-
if err != nil {
448-
return nil, err
449-
}
450-
if msg.Data != nil {
451-
msgs = append(msgs, msg)
452-
}
453-
}
454-
455-
return msgs, nil
456-
}
457-
458-
func nextMsg(
459-
shares [][]byte,
460-
current,
461-
nid []byte,
462-
cursor,
463-
msgLen uint64,
464-
) ([]byte, []byte, uint64, uint64, Message, error) {
465-
switch {
466-
// the message uses all of the current share data and at least some of the
467-
// next share
468-
case msgLen > uint64(len(current)):
469-
// add the next share to the current one and try again
470-
cursor++
471-
current = append(current, shares[cursor][NamespaceSize:]...)
472-
return nextMsg(shares, current, nid, cursor, msgLen)
473-
474-
// the msg we're looking for is contained in the current share
475-
case msgLen <= uint64(len(current)):
476-
msg := Message{nid, current[:msgLen]}
477-
cursor++
478-
479-
// call it a day if the work is done
480-
if cursor >= uint64(len(shares)) {
481-
return nil, nil, cursor, 0, msg, nil
482-
}
483-
484-
nextNid := shares[cursor][:NamespaceSize]
485-
next, msgLen, err := parseDelimiter(shares[cursor][NamespaceSize:])
486-
return next, nextNid, cursor, msgLen, msg, err
487-
}
488-
// this code is unreachable but the compiler doesn't know that
489-
return nil, nil, 0, 0, MessageEmpty, nil
490-
}
491-
492-
// parseDelimiter finds and returns the length delimiter of the message provided
493-
// while also removing the delimiter bytes from the input
494-
func parseDelimiter(input []byte) ([]byte, uint64, error) {
495-
if len(input) == 0 {
496-
return input, 0, nil
497-
}
498-
499-
l := binary.MaxVarintLen64
500-
if len(input) < binary.MaxVarintLen64 {
501-
l = len(input)
502-
}
503-
504-
delimiter := zeroPadIfNecessary(input[:l], binary.MaxVarintLen64)
505-
506-
// read the length of the message
507-
r := bytes.NewBuffer(delimiter)
508-
msgLen, err := binary.ReadUvarint(r)
509-
if err != nil {
510-
return nil, 0, err
511-
}
512-
513-
// calculate the number of bytes used by the delimiter
514-
lenBuf := make([]byte, binary.MaxVarintLen64)
515-
n := binary.PutUvarint(lenBuf, msgLen)
516-
517-
// return the input without the length delimiter
518-
return input[n:], msgLen, nil
519-
}

0 commit comments

Comments
 (0)
Please sign in to comment.