Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Spec compliant merge shares #261

Merged
merged 35 commits into from
Apr 6, 2021
Merged
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
494db15
start spec compliant share merging
evan-forbes Mar 31, 2021
0d15df4
refactor and finish unit testing
evan-forbes Apr 1, 2021
3f0d808
whoops
evan-forbes Apr 1, 2021
8535122
linter gods
evan-forbes Apr 1, 2021
8bae88b
fix initial changes and use constants
evan-forbes Apr 1, 2021
cbb5d76
use constant
evan-forbes Apr 1, 2021
b523831
more polish
evan-forbes Apr 1, 2021
f0f38a6
docs fix
evan-forbes Apr 1, 2021
014a00e
review feedback: docs and out of range panic protection
evan-forbes Apr 1, 2021
0896a3b
review feedback: add panic protection from empty input
evan-forbes Apr 1, 2021
96eafc7
use constant instead of recalculating `ShareSize`
evan-forbes Apr 1, 2021
c3d897a
don't redeclare existing var
evan-forbes Apr 1, 2021
28f9769
be more explicit with returned nil
evan-forbes Apr 1, 2021
9ff16f5
use constant instead of recalculating `ShareSize`
evan-forbes Apr 1, 2021
39ce26a
review feedback: use consistent capitalization
evan-forbes Apr 1, 2021
8794411
stop accepting reserved namespaces as normal messages
evan-forbes Apr 1, 2021
2b28c59
use a descriptive var name for message length
evan-forbes Apr 1, 2021
d118716
linter and comparison fix
evan-forbes Apr 1, 2021
3ae1f79
reorg tests, add test for parse delimiter, DataFromBlock and fix evid…
evan-forbes Apr 2, 2021
dfe2a07
catch error for linter
evan-forbes Apr 3, 2021
e9f3a2e
update test MakeShares to include length delimiters for the SHARE_RES…
evan-forbes Apr 3, 2021
221b0fb
minor iteration change
evan-forbes Apr 3, 2021
37316e8
refactor share splitting to fix bug
evan-forbes Apr 3, 2021
e3736d6
fix all bugs with third and final refactor
evan-forbes Apr 4, 2021
632bb37
Merge branch 'master' into evan/merge-shares
evan-forbes Apr 4, 2021
3ca2afa
fix conflict
evan-forbes Apr 4, 2021
0f930fd
revert unnecessary changes
evan-forbes Apr 4, 2021
91c3989
review feedback: better docs
evan-forbes Apr 5, 2021
043812d
reivew feedback: add comment for safeLen
evan-forbes Apr 5, 2021
76d7a4b
review feedback: remove unnecessay comments
evan-forbes Apr 5, 2021
1cb4030
review feedback: split up share merging and splitting into their own …
evan-forbes Apr 5, 2021
a88db3b
review feedback: more descriptive var names
evan-forbes Apr 5, 2021
2aad8fd
fix accidental change
evan-forbes Apr 5, 2021
887aa08
add some constant docs
evan-forbes Apr 5, 2021
40a57c1
spelling error
evan-forbes Apr 5, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
revert unnecessary changes
evan-forbes committed Apr 4, 2021

Verified

This commit was signed with the committer’s verified signature.
evan-forbes Evan Forbes
commit 0f930fddcbf4bd68e7f31592a4a9621f461043ba
102 changes: 48 additions & 54 deletions types/shares.go
Original file line number Diff line number Diff line change
@@ -107,69 +107,63 @@ func splitMessage(rawData []byte, nid namespace.ID) []NamespacedShare {
return shares
}

// splitContiguous fits the provided raw length delimited transactions,
// intermediate state roots, or evidence into the minimum number of namespaced
// shares.
func splitContiguous(nid namespace.ID, rawTxs [][]byte) []NamespacedShare {
// splitContiguous splits multiple raw data contiguously as shares.
// Used for transactions, intermediate state roots, and evidence.
func splitContiguous(nid namespace.ID, rawDatas [][]byte) []NamespacedShare {
shares := make([]NamespacedShare, 0)
if len(rawTxs) == 0 {
return nil
}
// set start index == 0 by preppending an empty slice
rawTxs = append(append(
make([][]byte, 0, len(rawTxs)+1),
[]byte{}),
rawTxs...)

for len(rawTxs) > 0 {
var share NamespacedShare
share, rawTxs = mintShare(nid, rawTxs)
// Index into the outer slice of rawDatas
outerIndex := 0
// Index into the inner slice of rawDatas
innerIndex := 0
for outerIndex < len(rawDatas) {
var rawData []byte
startIndex := 0
rawData, outerIndex, innerIndex, startIndex = getNextChunk(rawDatas, outerIndex, innerIndex, TxShareSize)
rawShare := append(append(append(
make([]byte, 0, len(nid)+1+len(rawData)),
nid...),
byte(startIndex)),
rawData...)
paddedShare := zeroPadIfNecessary(rawShare, ShareSize)
share := NamespacedShare{paddedShare, nid}
shares = append(shares, share)
}

return shares
}

// mintShare creates a single share using as many transactions as possible.
// Transactions are broken apart to maximize share space usage
func mintShare(nid namespace.ID, rawTxs [][]byte) (NamespacedShare, [][]byte) {
reservedStartIndex := len(rawTxs[0])
rawData := make([]byte, 0, TxShareSize)
outTxs := rawTxs

// add as many txs as possible to the share
for i := 0; i < len(rawTxs); i++ {
tx := rawTxs[i]

// add the tx if we still have room in the share
if len(rawData)+len(tx) <= TxShareSize {
rawData = append(rawData, tx...)
outTxs = outTxs[1:]
continue
// getNextChunk gets the next chunk for contiguous shares
// Precondition: none of the slices in rawDatas is zero-length
// This precondition should always hold at this point since zero-length txs are simply invalid.
func getNextChunk(rawDatas [][]byte, outerIndex int, innerIndex int, width int) ([]byte, int, int, int) {
rawData := make([]byte, 0, width)
startIndex := 0
firstBytesToFetch := 0

curIndex := 0
for curIndex < width && outerIndex < len(rawDatas) {
bytesToFetch := min(len(rawDatas[outerIndex])-innerIndex, width-curIndex)
if bytesToFetch == 0 {
panic("zero-length contiguous share data is invalid")
}

// If we don't have room, fill remaining data in the share with as much
// of the tx as possible
remaining := TxShareSize - len(rawData)

rawData = append(rawData, tx[:remaining]...)

// remove the portion of the tx that was added in this share
outTxs[0] = tx[remaining:]

break
if curIndex == 0 {
firstBytesToFetch = bytesToFetch
}
// If we've already placed some data in this chunk, that means
// a new data segment begins
if curIndex != 0 {
// Offset by the fixed reserved bytes at the beginning of the share
startIndex = firstBytesToFetch + NamespaceSize + ShareReservedBytes
}
rawData = append(rawData, rawDatas[outerIndex][innerIndex:innerIndex+bytesToFetch]...)
innerIndex += bytesToFetch
if innerIndex >= len(rawDatas[outerIndex]) {
innerIndex = 0
outerIndex++
}
curIndex += bytesToFetch
}

rawData = zeroPadIfNecessary(rawData, TxShareSize)

// assemble the new share
share := append(append(append(
make([]byte, 0, ShareSize),
nid...),
byte(reservedStartIndex)),
rawData...)

return NamespacedShare{share, nid}, outTxs
return rawData, outerIndex, innerIndex, startIndex
}

func GenerateTailPaddingShares(n int, shareWidth int) NamespacedShares {
6 changes: 3 additions & 3 deletions types/shares_test.go
Original file line number Diff line number Diff line change
@@ -72,7 +72,7 @@ func TestMakeShares(t *testing.T) {
ID: reservedEvidenceNamespaceID,
}, NamespacedShare{
Share: append(
append(reservedEvidenceNamespaceID, byte(131)),
append(reservedEvidenceNamespaceID, byte(0)),
zeroPadIfNecessary(testEvidenceBytes[TxShareSize:], TxShareSize)...,
),
ID: reservedEvidenceNamespaceID,
@@ -106,7 +106,7 @@ func TestMakeShares(t *testing.T) {
},
NamespacedShare{
Share: append(
append(reservedTxNamespaceID, byte(155)),
append(reservedTxNamespaceID, byte(0)),
zeroPadIfNecessary(largeTxLenDelimited[TxShareSize:], TxShareSize)...,
),
ID: reservedTxNamespaceID,
@@ -127,7 +127,7 @@ func TestMakeShares(t *testing.T) {
},
NamespacedShare{
Share: append(
append(reservedTxNamespaceID, byte(len(largeTxLenDelimited)-TxShareSize)),
append(reservedTxNamespaceID, byte(len(largeTxLenDelimited)-TxShareSize+NamespaceSize+ShareReservedBytes)),
zeroPadIfNecessary(
append(largeTxLenDelimited[TxShareSize:], smolTxLenDelimited...),
TxShareSize,