Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 317d838

Browse files
evan-forbesHlib Kanunnikovadlerjohnliamsi
committedApr 25, 2022
Spec compliant merge shares (#261)
* start spec compliant share merging * refactor and finish unit testing * whoops * linter gods * fix initial changes and use constants * use constant * more polish * docs fix* review feedback: docs and out of range panic protection * review feedback: add panic protection from empty input * use constant instead of recalculating `ShareSize`* don't redeclare existing var* be more explicit with returned nil* use constant instead of recalculating `ShareSize`* review feedback: use consistent capitalization * stop accepting reserved namespaces as normal messages * use a descriptive var name for message length * linter and comparison fix * reorg tests, add test for parse delimiter, DataFromBlock and fix evidence marshal bug * catch error for linter * update test MakeShares to include length delimiters for the SHARE_RESERVED_BYTE * minor iteration change * refactor share splitting to fix bug * fix all bugs with third and final refactor * fix conflict * revert unnecessary changes * review feedback: better docs* reivew feedback: add comment for safeLen * review feedback: remove unnecessay comments * review feedback: split up share merging and splitting into their own files * review feedback: more descriptive var names * fix accidental change * add some constant docs * spelling error Co-authored-by: Hlib Kanunnikov <[email protected]> Co-authored-by: John Adler <[email protected]> Co-authored-by: Ismail Khoffi <[email protected]>
1 parent 75b0a7d commit 317d838

File tree

7 files changed

+944
-154
lines changed

7 files changed

+944
-154
lines changed
 

‎pkg/consts/consts.go

+3
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"crypto/sha256"
55

66
"github.com/celestiaorg/nmt/namespace"
7+
"github.com/celestiaorg/rsmt2d"
78
)
89

910
// This contains all constants of:
@@ -61,4 +62,6 @@ var (
6162

6263
// NewBaseHashFunc change accordingly if another hash.Hash should be used as a base hasher in the NMT:
6364
NewBaseHashFunc = sha256.New
65+
66+
DefaultCodec = rsmt2d.NewRSGF8Codec
6467
)

‎types/block.go

+81
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/tendermint/tendermint/crypto"
1616
"github.com/tendermint/tendermint/crypto/merkle"
1717
"github.com/tendermint/tendermint/crypto/tmhash"
18+
"github.com/tendermint/tendermint/internal/libs/protoio"
1819
tmsync "github.com/tendermint/tendermint/internal/libs/sync"
1920
"github.com/tendermint/tendermint/libs/bits"
2021
tmbytes "github.com/tendermint/tendermint/libs/bytes"
@@ -1053,6 +1054,69 @@ type Data struct {
10531054
Messages Messages `json:"msgs"`
10541055
}
10551056

1057+
// ComputeShares splits block data into shares of an original data square and
1058+
// returns them along with an amount of non-redundant shares. The shares
1059+
// returned are padded to complete a square size that is a power of two
1060+
func (data *Data) ComputeShares() (NamespacedShares, int) {
1061+
// TODO(ismail): splitting into shares should depend on the block size and layout
1062+
// see: https://github.com/celestiaorg/celestia-specs/blob/master/specs/block_proposer.md#laying-out-transactions-and-messages
1063+
1064+
// reserved shares:
1065+
txShares := data.Txs.SplitIntoShares()
1066+
intermRootsShares := data.IntermediateStateRoots.SplitIntoShares()
1067+
evidenceShares := data.Evidence.SplitIntoShares()
1068+
1069+
// application data shares from messages:
1070+
msgShares := data.Messages.SplitIntoShares()
1071+
curLen := len(txShares) + len(intermRootsShares) + len(evidenceShares) + len(msgShares)
1072+
1073+
// find the number of shares needed to create a square that has a power of
1074+
// two width
1075+
wantLen := paddedLen(curLen)
1076+
1077+
// ensure that the min square size is used
1078+
if wantLen < consts.MinSharecount {
1079+
wantLen = consts.MinSharecount
1080+
}
1081+
1082+
tailShares := TailPaddingShares(wantLen - curLen)
1083+
1084+
return append(append(append(append(
1085+
txShares,
1086+
intermRootsShares...),
1087+
evidenceShares...),
1088+
msgShares...),
1089+
tailShares...), curLen
1090+
}
1091+
1092+
// paddedLen calculates the number of shares needed to make a power of 2 square
1093+
// given the current number of shares
1094+
func paddedLen(length int) int {
1095+
width := uint32(math.Ceil(math.Sqrt(float64(length))))
1096+
width = nextHighestPowerOf2(width)
1097+
return int(width * width)
1098+
}
1099+
1100+
// nextPowerOf2 returns the next highest power of 2 unless the input is a power
1101+
// of two, in which case it returns the input
1102+
func nextHighestPowerOf2(v uint32) uint32 {
1103+
if v == 0 {
1104+
return 0
1105+
}
1106+
1107+
// find the next highest power using bit mashing
1108+
v--
1109+
v |= v >> 1
1110+
v |= v >> 2
1111+
v |= v >> 4
1112+
v |= v >> 8
1113+
v |= v >> 16
1114+
v++
1115+
1116+
// return the next highest power
1117+
return v
1118+
}
1119+
10561120
type Messages struct {
10571121
MessagesList []Message `json:"msgs"`
10581122
}
@@ -1320,6 +1384,23 @@ func (data *EvidenceData) FromProto(eviData *tmproto.EvidenceList) error {
13201384
return nil
13211385
}
13221386

1387+
func (data *EvidenceData) SplitIntoShares() NamespacedShares {
1388+
rawDatas := make([][]byte, 0, len(data.Evidence))
1389+
for _, ev := range data.Evidence {
1390+
pev, err := EvidenceToProto(ev)
1391+
if err != nil {
1392+
panic("failure to convert evidence to equivalent proto type")
1393+
}
1394+
rawData, err := protoio.MarshalDelimited(pev)
1395+
if err != nil {
1396+
panic(err)
1397+
}
1398+
rawDatas = append(rawDatas, rawData)
1399+
}
1400+
shares := splitContiguous(consts.EvidenceNamespaceID, rawDatas)
1401+
return shares
1402+
}
1403+
13231404
//--------------------------------------------------------------------------------
13241405

13251406
// BlockID

‎types/share_merging.go

+333
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
package types
2+
3+
import (
4+
"bytes"
5+
"encoding/binary"
6+
"errors"
7+
8+
"github.com/celestiaorg/rsmt2d"
9+
"github.com/gogo/protobuf/proto"
10+
tmbytes "github.com/tendermint/tendermint/libs/bytes"
11+
"github.com/tendermint/tendermint/pkg/consts"
12+
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
13+
)
14+
15+
// DataFromSquare extracts block data from an extended data square.
16+
func DataFromSquare(eds *rsmt2d.ExtendedDataSquare) (Data, error) {
17+
originalWidth := eds.Width() / 2
18+
19+
// sort block data shares by namespace
20+
var (
21+
sortedTxShares [][]byte
22+
sortedISRShares [][]byte
23+
sortedEvdShares [][]byte
24+
sortedMsgShares [][]byte
25+
)
26+
27+
// iterate over each row index
28+
for x := uint(0); x < originalWidth; x++ {
29+
// iterate over each share in the original data square
30+
row := eds.Row(x)
31+
32+
for _, share := range row[:originalWidth] {
33+
// sort the data of that share types via namespace
34+
nid := share[:consts.NamespaceSize]
35+
switch {
36+
case bytes.Equal(consts.TxNamespaceID, nid):
37+
sortedTxShares = append(sortedTxShares, share)
38+
39+
case bytes.Equal(consts.IntermediateStateRootsNamespaceID, nid):
40+
sortedISRShares = append(sortedISRShares, share)
41+
42+
case bytes.Equal(consts.EvidenceNamespaceID, nid):
43+
sortedEvdShares = append(sortedEvdShares, share)
44+
45+
case bytes.Equal(consts.TailPaddingNamespaceID, nid):
46+
continue
47+
48+
// ignore unused but reserved namespaces
49+
case bytes.Compare(nid, consts.MaxReservedNamespace) < 1:
50+
continue
51+
52+
// every other namespaceID should be a message
53+
default:
54+
sortedMsgShares = append(sortedMsgShares, share)
55+
}
56+
}
57+
}
58+
59+
// pass the raw share data to their respective parsers
60+
txs, err := parseTxs(sortedTxShares)
61+
if err != nil {
62+
return Data{}, err
63+
}
64+
65+
isrs, err := parseISRs(sortedISRShares)
66+
if err != nil {
67+
return Data{}, err
68+
}
69+
70+
evd, err := parseEvd(sortedEvdShares)
71+
if err != nil {
72+
return Data{}, err
73+
}
74+
75+
msgs, err := parseMsgs(sortedMsgShares)
76+
if err != nil {
77+
return Data{}, err
78+
}
79+
80+
return Data{
81+
Txs: txs,
82+
IntermediateStateRoots: isrs,
83+
Evidence: evd,
84+
Messages: msgs,
85+
}, nil
86+
}
87+
88+
// parseTxs collects all of the transactions from the shares provided
89+
func parseTxs(shares [][]byte) (Txs, error) {
90+
// parse the sharse
91+
rawTxs, err := processContiguousShares(shares)
92+
if err != nil {
93+
return nil, err
94+
}
95+
96+
// convert to the Tx type
97+
txs := make(Txs, len(rawTxs))
98+
for i := 0; i < len(txs); i++ {
99+
txs[i] = Tx(rawTxs[i])
100+
}
101+
102+
return txs, nil
103+
}
104+
105+
// parseISRs collects all the intermediate state roots from the shares provided
106+
func parseISRs(shares [][]byte) (IntermediateStateRoots, error) {
107+
rawISRs, err := processContiguousShares(shares)
108+
if err != nil {
109+
return IntermediateStateRoots{}, err
110+
}
111+
112+
ISRs := make([]tmbytes.HexBytes, len(rawISRs))
113+
for i := 0; i < len(ISRs); i++ {
114+
ISRs[i] = rawISRs[i]
115+
}
116+
117+
return IntermediateStateRoots{RawRootsList: ISRs}, nil
118+
}
119+
120+
// parseEvd collects all evidence from the shares provided.
121+
func parseEvd(shares [][]byte) (EvidenceData, error) {
122+
// the raw data returned does not have length delimiters or namespaces and
123+
// is ready to be unmarshaled
124+
rawEvd, err := processContiguousShares(shares)
125+
if err != nil {
126+
return EvidenceData{}, err
127+
}
128+
129+
evdList := make(EvidenceList, len(rawEvd))
130+
131+
// parse into protobuf bytes
132+
for i := 0; i < len(rawEvd); i++ {
133+
// unmarshal the evidence
134+
var protoEvd tmproto.Evidence
135+
err := proto.Unmarshal(rawEvd[i], &protoEvd)
136+
if err != nil {
137+
return EvidenceData{}, err
138+
}
139+
evd, err := EvidenceFromProto(&protoEvd)
140+
if err != nil {
141+
return EvidenceData{}, err
142+
}
143+
144+
evdList[i] = evd
145+
}
146+
147+
return EvidenceData{Evidence: evdList}, nil
148+
}
149+
150+
// parseMsgs collects all messages from the shares provided
151+
func parseMsgs(shares [][]byte) (Messages, error) {
152+
msgList, err := parseMsgShares(shares)
153+
if err != nil {
154+
return Messages{}, err
155+
}
156+
157+
return Messages{
158+
MessagesList: msgList,
159+
}, nil
160+
}
161+
162+
// processContiguousShares takes raw shares and extracts out transactions,
163+
// intermediate state roots, or evidence. The returned [][]byte do have
164+
// namespaces or length delimiters and are ready to be unmarshalled
165+
func processContiguousShares(shares [][]byte) (txs [][]byte, err error) {
166+
if len(shares) == 0 {
167+
return nil, nil
168+
}
169+
170+
ss := newShareStack(shares)
171+
return ss.resolve()
172+
}
173+
174+
// shareStack hold variables for peel
175+
type shareStack struct {
176+
shares [][]byte
177+
txLen uint64
178+
txs [][]byte
179+
cursor int
180+
}
181+
182+
func newShareStack(shares [][]byte) *shareStack {
183+
return &shareStack{shares: shares}
184+
}
185+
186+
func (ss *shareStack) resolve() ([][]byte, error) {
187+
if len(ss.shares) == 0 {
188+
return nil, nil
189+
}
190+
err := ss.peel(ss.shares[0][consts.NamespaceSize+consts.ShareReservedBytes:], true)
191+
return ss.txs, err
192+
}
193+
194+
// peel recursively parses each chunk of data (either a transaction,
195+
// intermediate state root, or evidence) and adds it to the underlying slice of data.
196+
func (ss *shareStack) peel(share []byte, delimited bool) (err error) {
197+
if delimited {
198+
var txLen uint64
199+
share, txLen, err = parseDelimiter(share)
200+
if err != nil {
201+
return err
202+
}
203+
if txLen == 0 {
204+
return nil
205+
}
206+
ss.txLen = txLen
207+
}
208+
// safeLen describes the point in the share where it can be safely split. If
209+
// split beyond this point, it is possible to break apart a length
210+
// delimiter, which will result in incorrect share merging
211+
safeLen := len(share) - binary.MaxVarintLen64
212+
if safeLen < 0 {
213+
safeLen = 0
214+
}
215+
if ss.txLen <= uint64(safeLen) {
216+
ss.txs = append(ss.txs, share[:ss.txLen])
217+
share = share[ss.txLen:]
218+
return ss.peel(share, true)
219+
}
220+
// add the next share to the current share to continue merging if possible
221+
if len(ss.shares) > ss.cursor+1 {
222+
ss.cursor++
223+
share := append(share, ss.shares[ss.cursor][consts.NamespaceSize+consts.ShareReservedBytes:]...)
224+
return ss.peel(share, false)
225+
}
226+
// collect any remaining data
227+
if ss.txLen <= uint64(len(share)) {
228+
ss.txs = append(ss.txs, share[:ss.txLen])
229+
share = share[ss.txLen:]
230+
return ss.peel(share, true)
231+
}
232+
return errors.New("failure to parse block data: transaction length exceeded data length")
233+
}
234+
235+
// parseMsgShares iterates through raw shares and separates the contiguous chunks
236+
// of data. It is only used for Messages, i.e. shares with a non-reserved namespace.
237+
func parseMsgShares(shares [][]byte) ([]Message, error) {
238+
if len(shares) == 0 {
239+
return nil, nil
240+
}
241+
242+
// set the first nid and current share
243+
nid := shares[0][:consts.NamespaceSize]
244+
currentShare := shares[0][consts.NamespaceSize:]
245+
// find and remove the msg len delimiter
246+
currentShare, msgLen, err := parseDelimiter(currentShare)
247+
if err != nil {
248+
return nil, err
249+
}
250+
251+
var msgs []Message
252+
for cursor := uint64(0); cursor < uint64(len(shares)); {
253+
var msg Message
254+
currentShare, nid, cursor, msgLen, msg, err = nextMsg(
255+
shares,
256+
currentShare,
257+
nid,
258+
cursor,
259+
msgLen,
260+
)
261+
if err != nil {
262+
return nil, err
263+
}
264+
if msg.Data != nil {
265+
msgs = append(msgs, msg)
266+
}
267+
}
268+
269+
return msgs, nil
270+
}
271+
272+
func nextMsg(
273+
shares [][]byte,
274+
current,
275+
nid []byte,
276+
cursor,
277+
msgLen uint64,
278+
) ([]byte, []byte, uint64, uint64, Message, error) {
279+
switch {
280+
// the message uses all of the current share data and at least some of the
281+
// next share
282+
case msgLen > uint64(len(current)):
283+
// add the next share to the current one and try again
284+
cursor++
285+
current = append(current, shares[cursor][consts.NamespaceSize:]...)
286+
return nextMsg(shares, current, nid, cursor, msgLen)
287+
288+
// the msg we're looking for is contained in the current share
289+
case msgLen <= uint64(len(current)):
290+
msg := Message{nid, current[:msgLen]}
291+
cursor++
292+
293+
// call it a day if the work is done
294+
if cursor >= uint64(len(shares)) {
295+
return nil, nil, cursor, 0, msg, nil
296+
}
297+
298+
nextNid := shares[cursor][:consts.NamespaceSize]
299+
next, msgLen, err := parseDelimiter(shares[cursor][consts.NamespaceSize:])
300+
return next, nextNid, cursor, msgLen, msg, err
301+
}
302+
// this code is unreachable but the compiler doesn't know that
303+
return nil, nil, 0, 0, Message{}, nil
304+
}
305+
306+
// parseDelimiter finds and returns the length delimiter of the message provided
307+
// while also removing the delimiter bytes from the input
308+
func parseDelimiter(input []byte) ([]byte, uint64, error) {
309+
if len(input) == 0 {
310+
return input, 0, nil
311+
}
312+
313+
l := binary.MaxVarintLen64
314+
if len(input) < binary.MaxVarintLen64 {
315+
l = len(input)
316+
}
317+
318+
delimiter := zeroPadIfNecessary(input[:l], binary.MaxVarintLen64)
319+
320+
// read the length of the message
321+
r := bytes.NewBuffer(delimiter)
322+
msgLen, err := binary.ReadUvarint(r)
323+
if err != nil {
324+
return nil, 0, err
325+
}
326+
327+
// calculate the number of bytes used by the delimiter
328+
lenBuf := make([]byte, binary.MaxVarintLen64)
329+
n := binary.PutUvarint(lenBuf, msgLen)
330+
331+
// return the input without the length delimiter
332+
return input[n:], msgLen, nil
333+
}

‎types/share_splitting.go

+148
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package types
2+
3+
import (
4+
"bytes"
5+
6+
"github.com/celestiaorg/nmt/namespace"
7+
"github.com/tendermint/tendermint/pkg/consts"
8+
)
9+
10+
// appendToShares appends raw data as shares.
11+
// Used for messages.
12+
func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare {
13+
if len(rawData) <= consts.MsgShareSize {
14+
rawShare := append(append(
15+
make([]byte, 0, len(nid)+len(rawData)),
16+
nid...),
17+
rawData...,
18+
)
19+
paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize)
20+
share := NamespacedShare{paddedShare, nid}
21+
shares = append(shares, share)
22+
} else { // len(rawData) > MsgShareSize
23+
shares = append(shares, splitMessage(rawData, nid)...)
24+
}
25+
return shares
26+
}
27+
28+
// splitMessage breaks the data in a message into the minimum number of
29+
// namespaced shares
30+
func splitMessage(rawData []byte, nid namespace.ID) []NamespacedShare {
31+
shares := make([]NamespacedShare, 0)
32+
firstRawShare := append(append(
33+
make([]byte, 0, consts.ShareSize),
34+
nid...),
35+
rawData[:consts.MsgShareSize]...,
36+
)
37+
shares = append(shares, NamespacedShare{firstRawShare, nid})
38+
rawData = rawData[consts.MsgShareSize:]
39+
for len(rawData) > 0 {
40+
shareSizeOrLen := min(consts.MsgShareSize, len(rawData))
41+
rawShare := append(append(
42+
make([]byte, 0, consts.ShareSize),
43+
nid...),
44+
rawData[:shareSizeOrLen]...,
45+
)
46+
paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize)
47+
share := NamespacedShare{paddedShare, nid}
48+
shares = append(shares, share)
49+
rawData = rawData[shareSizeOrLen:]
50+
}
51+
return shares
52+
}
53+
54+
// splitContiguous splits multiple raw data contiguously as shares.
55+
// Used for transactions, intermediate state roots, and evidence.
56+
func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
57+
shares := make([]NamespacedShare, 0)
58+
// Index into the outer slice of rawDatas
59+
outerIndex := 0
60+
// Index into the inner slice of rawDatas
61+
innerIndex := 0
62+
for outerIndex < len(rawDatas) {
63+
var rawData []byte
64+
startIndex := 0
65+
rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, consts.TxShareSize)
66+
rawShare := append(append(append(
67+
make([]byte, 0, len(nid)+1+len(rawData)),
68+
nid...),
69+
byte(startIndex)),
70+
rawData...)
71+
paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize)
72+
share := NamespacedShare{paddedShare, nid}
73+
shares = append(shares, share)
74+
}
75+
return shares
76+
}
77+
78+
// getNextChunk gets the next chunk for contiguous shares
79+
// Precondition: none of the slices in rawDatas is zero-length
80+
// This precondition should always hold at this point since zero-length txs are simply invalid.
81+
func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) {
82+
rawData := make([]byte, 0, width)
83+
startIndex := 0
84+
firstBytesToFetch := 0
85+
86+
curIndex := 0
87+
for curIndex < width && outerIndex < len(rawDatas) {
88+
bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex, width-curIndex)
89+
if bytesToFetch == 0 {
90+
panic("zero-length contiguous share data is invalid")
91+
}
92+
if curIndex == 0 {
93+
firstBytesToFetch = bytesToFetch
94+
}
95+
// If we've already placed some data in this chunk, that means
96+
// a new data segment begins
97+
if curIndex != 0 {
98+
// Offset by the fixed reserved bytes at the beginning of the share
99+
startIndex = firstBytesToFetch + consts.NamespaceSize + consts.ShareReservedBytes
100+
}
101+
rawData = append(rawData, rawDatas[outerIndex][innerIndex:innerIndex+bytesToFetch]...)
102+
innerIndex += bytesToFetch
103+
if innerIndex >= len(rawDatas[outerIndex]) {
104+
innerIndex = 0
105+
outerIndex++
106+
}
107+
curIndex += bytesToFetch
108+
}
109+
110+
return rawData, outerIndex, innerIndex, startIndex
111+
}
112+
113+
// tail is filler for all tail padded shares
114+
// it is allocated once and used everywhere
115+
var tailPaddingShare = append(
116+
append(make([]byte, 0, consts.ShareSize), consts.TailPaddingNamespaceID...),
117+
bytes.Repeat([]byte{0}, consts.ShareSize-consts.NamespaceSize)...,
118+
)
119+
120+
func TailPaddingShares(n int) NamespacedShares {
121+
shares := make([]NamespacedShare, n)
122+
for i := 0; i < n; i++ {
123+
shares[i] = NamespacedShare{
124+
Share: tailPaddingShare,
125+
ID: consts.TailPaddingNamespaceID,
126+
}
127+
}
128+
return shares
129+
}
130+
131+
func min(a, b int) int {
132+
if a <= b {
133+
return a
134+
}
135+
return b
136+
}
137+
138+
func zeroPadIfNecessary(share []byte, width int) []byte {
139+
oldLen := len(share)
140+
if oldLen < width {
141+
missingBytes := width - oldLen
142+
padByte := []byte{0}
143+
padding := bytes.Repeat(padByte, missingBytes)
144+
share = append(share, padding...)
145+
return share
146+
}
147+
return share
148+
}

‎types/shares.go

-122
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,15 @@
11
package types
22

33
import (
4-
"bytes"
54
"encoding/binary"
65

76
"github.com/celestiaorg/nmt/namespace"
8-
"github.com/tendermint/tendermint/pkg/consts"
97
)
108

119
// Share contains the raw share data without the corresponding namespace.
1210
type Share []byte
1311

1412
// NamespacedShare extends a Share with the corresponding namespace.
15-
// It implements the namespace.Data interface and hence can be used
16-
// for pushing the shares to the namespaced Merkle tree.
1713
type NamespacedShare struct {
1814
Share
1915
ID namespace.ID
@@ -45,7 +41,6 @@ func (tx Tx) MarshalDelimited() ([]byte, error) {
4541
lenBuf := make([]byte, binary.MaxVarintLen64)
4642
length := uint64(len(tx))
4743
n := binary.PutUvarint(lenBuf, length)
48-
4944
return append(lenBuf[:n], tx...), nil
5045
}
5146

@@ -55,122 +50,5 @@ func (m Message) MarshalDelimited() ([]byte, error) {
5550
lenBuf := make([]byte, binary.MaxVarintLen64)
5651
length := uint64(len(m.Data))
5752
n := binary.PutUvarint(lenBuf, length)
58-
5953
return append(lenBuf[:n], m.Data...), nil
6054
}
61-
62-
// appendToShares appends raw data as shares.
63-
// Used for messages.
64-
func appendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare {
65-
if len(rawData) <= consts.MsgShareSize {
66-
rawShare := []byte(append(nid, rawData...))
67-
paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize)
68-
share := NamespacedShare{paddedShare, nid}
69-
shares = append(shares, share)
70-
} else { // len(rawData) > MsgShareSize
71-
shares = append(shares, split(rawData, nid)...)
72-
}
73-
return shares
74-
}
75-
76-
// splitContiguous splits multiple raw data contiguously as shares.
77-
// Used for transactions, intermediate state roots, and evidence.
78-
func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
79-
shares := make([]NamespacedShare, 0)
80-
// Index into the outer slice of rawDatas
81-
outerIndex := 0
82-
// Index into the inner slice of rawDatas
83-
innerIndex := 0
84-
for outerIndex < len(rawDatas) {
85-
var rawData []byte
86-
startIndex := 0
87-
rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, consts.TxShareSize)
88-
rawShare := []byte(append(append(nid, byte(startIndex)), rawData...))
89-
paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize)
90-
share := NamespacedShare{paddedShare, nid}
91-
shares = append(shares, share)
92-
}
93-
return shares
94-
}
95-
96-
// TODO(ismail): implement corresponding merge method for clients requesting
97-
// shares for a particular namespace
98-
func split(rawData []byte, nid namespace.ID) []NamespacedShare {
99-
shares := make([]NamespacedShare, 0)
100-
firstRawShare := []byte(append(nid, rawData[:consts.MsgShareSize]...))
101-
shares = append(shares, NamespacedShare{firstRawShare, nid})
102-
rawData = rawData[consts.MsgShareSize:]
103-
for len(rawData) > 0 {
104-
shareSizeOrLen := min(consts.MsgShareSize, len(rawData))
105-
rawShare := make([]byte, consts.NamespaceSize)
106-
copy(rawShare, nid)
107-
rawShare = append(rawShare, rawData[:shareSizeOrLen]...)
108-
paddedShare := zeroPadIfNecessary(rawShare, consts.ShareSize)
109-
share := NamespacedShare{paddedShare, nid}
110-
shares = append(shares, share)
111-
rawData = rawData[shareSizeOrLen:]
112-
}
113-
return shares
114-
}
115-
116-
// getNextChunk gets the next chunk for contiguous shares
117-
// Precondition: none of the slices in rawDatas is zero-length
118-
// This precondition should always hold at this point since zero-length txs are simply invalid.
119-
func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) {
120-
rawData := make([]byte, 0, width)
121-
startIndex := 0
122-
firstBytesToFetch := 0
123-
124-
curIndex := 0
125-
for curIndex < width && outerIndex < len(rawDatas) {
126-
bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex, width-curIndex)
127-
if bytesToFetch == 0 {
128-
panic("zero-length contiguous share data is invalid")
129-
}
130-
if curIndex == 0 {
131-
firstBytesToFetch = bytesToFetch
132-
}
133-
// If we've already placed some data in this chunk, that means
134-
// a new data segment begins
135-
if curIndex != 0 {
136-
// Offset by the fixed reserved bytes at the beginning of the share
137-
startIndex = firstBytesToFetch + consts.NamespaceSize + consts.ShareReservedBytes
138-
}
139-
rawData = append(rawData, rawDatas[outerIndex][innerIndex:innerIndex+bytesToFetch]...)
140-
innerIndex += bytesToFetch
141-
if innerIndex >= len(rawDatas[outerIndex]) {
142-
innerIndex = 0
143-
outerIndex++
144-
}
145-
curIndex += bytesToFetch
146-
}
147-
148-
return rawData, outerIndex, innerIndex, startIndex
149-
}
150-
151-
func GenerateTailPaddingShares(n int, shareWidth int) NamespacedShares {
152-
shares := make([]NamespacedShare, n)
153-
for i := 0; i < n; i++ {
154-
shares[i] = NamespacedShare{bytes.Repeat([]byte{0}, shareWidth), consts.TailPaddingNamespaceID}
155-
}
156-
return shares
157-
}
158-
159-
func min(a, b int) int {
160-
if a <= b {
161-
return a
162-
}
163-
return b
164-
}
165-
166-
func zeroPadIfNecessary(share []byte, width int) []byte {
167-
oldLen := len(share)
168-
if oldLen < width {
169-
missingBytes := width - oldLen
170-
padByte := []byte{0}
171-
padding := bytes.Repeat(padByte, missingBytes)
172-
share = append(share, padding...)
173-
return share
174-
}
175-
return share
176-
}

‎types/shares_test.go

+365-32
Large diffs are not rendered by default.

‎types/tx.go

+14
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/tendermint/tendermint/crypto/merkle"
1010
"github.com/tendermint/tendermint/crypto/tmhash"
1111
tmbytes "github.com/tendermint/tendermint/libs/bytes"
12+
"github.com/tendermint/tendermint/pkg/consts"
1213
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
1314
)
1415

@@ -79,6 +80,19 @@ func (txs Txs) Proof(i int) TxProof {
7980
}
8081
}
8182

83+
func (txs Txs) SplitIntoShares() NamespacedShares {
84+
rawDatas := make([][]byte, len(txs))
85+
for i, tx := range txs {
86+
rawData, err := tx.MarshalDelimited()
87+
if err != nil {
88+
panic(fmt.Sprintf("included Tx in mem-pool that can not be encoded %v", tx))
89+
}
90+
rawDatas[i] = rawData
91+
}
92+
shares := splitContiguous(consts.TxNamespaceID, rawDatas)
93+
return shares
94+
}
95+
8296
// TxProof represents a Merkle proof of the presence of a transaction in the Merkle tree.
8397
type TxProof struct {
8498
RootHash tmbytes.HexBytes `json:"root_hash"`

0 commit comments

Comments
 (0)
Please sign in to comment.