feat!: increase share size to 512 bytes (celestiaorg#850)

Closes celestiaorg#825

This is ready for review but we may not merge it b/c we haven't made a
final decision on what the share size should be

Co-authored-by: evan-forbes <[email protected]>

Author: 2 people

app/estimate_square_size_test.go

@@ -26,9 +26,9 @@ func Test_estimateSquareSize(t *testing.T) {
 	tests := []test{
 		{"empty block minimum square size", 0, 0, 0, appconsts.MinSquareSize},
 		{"full block with only txs", 10000, 0, 0, appconsts.MaxSquareSize},
-		{"random small block square size 4", 0, 1, appconsts.SparseShareContentSize, 4},
+		{"random small block square size 2", 0, 1, appconsts.SparseShareContentSize, 2},
 		{"random small block square size 4", 0, 1, appconsts.SparseShareContentSize * 10, 4},
-		{"random small block w/ 10 normal txs square size 4", 10, 1, appconsts.SparseShareContentSize, 8},
+		{"random small block w/ 10 normal txs square size 4", 10, 1, appconsts.SparseShareContentSize, 4},
 		{"random small block square size 16", 0, 4, appconsts.SparseShareContentSize * 8, 16},
 		{"random medium block square size 32", 0, 50, appconsts.SparseShareContentSize * 4, 32},
 		{"full block max square size", 0, 8000, appconsts.SparseShareContentSize, appconsts.MaxSquareSize},

app/test/block_size_test.go

@@ -31,7 +31,7 @@ func TestIntegrationTestSuite(t *testing.T) {
 	cfg.EnableTMLogging = false
 	cfg.MinGasPrices = "0ucls"
 	cfg.NumValidators = 1
-	cfg.TimeoutCommit = time.Millisecond * 200
+	cfg.TimeoutCommit = time.Millisecond * 400
 	suite.Run(t, NewIntegrationTestSuite(cfg))
 }
 

pkg/appconsts/appconsts.go

@@ -13,7 +13,7 @@ import (
 // https://github.com/celestiaorg/celestia-specs/blob/master/src/specs/consensus.md#constants
 const (
 	// ShareSize is the size of a share in bytes.
-	ShareSize = 256
+	ShareSize = 512
 
 	// NamespaceSize is the namespace size in bytes.
 	NamespaceSize = 8
@@ -27,7 +27,7 @@ const (
 
 	// CompactShareReservedBytes is the number of bytes reserved for the location of
 	// the first unit (transaction, ISR, evidence) in a compact share.
-	CompactShareReservedBytes = 1
+	CompactShareReservedBytes = 2
 
 	// ContinuationCompactShareContentSize is the number of bytes usable for
 	// data in a continuation compact share. A continuation share is any

pkg/da/data_availability_header_test.go

@@ -25,8 +25,8 @@ func TestNilDataAvailabilityHeaderHashDoesntCrash(t *testing.T) {
 func TestMinDataAvailabilityHeader(t *testing.T) {
 	dah := MinDataAvailabilityHeader()
 	expectedHash := []byte{
-		0x7b, 0x57, 0x8b, 0x35, 0x1b, 0x1b, 0xb, 0xbd, 0x70, 0xbb, 0x35, 0x0, 0x19, 0xeb, 0xc9, 0x64,
-		0xc4, 0x4a, 0x14, 0xa, 0x37, 0xef, 0x71, 0x5b, 0x55, 0x2a, 0x7f, 0x8f, 0x31, 0x5a, 0xcd, 0x19,
+		0x6f, 0x52, 0xda, 0xc1, 0x65, 0x45, 0xe4, 0x57, 0x25, 0xbe, 0x6e, 0xa3, 0x2a, 0xed, 0x55, 0x26,
+		0x6e, 0x45, 0x3, 0x48, 0x0, 0xee, 0xe1, 0xd8, 0x7c, 0x94, 0x28, 0xf4, 0x84, 0x4e, 0xa4, 0x7a,
 	}
 	require.Equal(t, expectedHash, dah.hash)
 	require.NoError(t, dah.ValidateBasic())
@@ -47,31 +47,32 @@ func TestNewDataAvailabilityHeader(t *testing.T) {
 		{
 			name: "typical",
 			expectedHash: []byte{
-				0xfe, 0x9c, 0x6b, 0xd8, 0xe5, 0x7c, 0xd1, 0x5d, 0x1f, 0xd6, 0x55, 0x7e, 0x87, 0x7d, 0xd9, 0x7d,
-				0xdb, 0xf2, 0x66, 0xfa, 0x60, 0x24, 0x2d, 0xb3, 0xa0, 0x9c, 0x4f, 0x4e, 0x5b, 0x2a, 0x2c, 0x2a,
+				0x57, 0x71, 0xc6, 0x77, 0x2f, 0x32, 0x95, 0x73, 0xaa, 0xb8, 0x20, 0xd1, 0xbe, 0x4c, 0xc2, 0x21,
+				0x7d, 0x54, 0xb6, 0x7e, 0xf2, 0x4f, 0xbc, 0xd3, 0x9a, 0x95, 0x15, 0xd0, 0x92, 0x63, 0xc1, 0xf9,
 			},
 			squareSize: 2,
 			shares:     generateShares(4, 1),
 		},
 		{
 			name: "max square size",
 			expectedHash: []byte{
-				0xe2, 0x87, 0x23, 0xd0, 0x2d, 0x54, 0x25, 0x5f, 0x79, 0x43, 0x8e, 0xfb, 0xb7, 0xe8, 0xfa, 0xf5,
-				0xbf, 0x93, 0x50, 0xb3, 0x64, 0xd0, 0x4f, 0xa7, 0x7b, 0xb1, 0x83, 0x3b, 0x8, 0xba, 0xd3, 0xa4,
+				0xbf, 0xe5, 0x8f, 0x4b, 0xae, 0x2b, 0x65, 0x8b, 0xa8, 0xcb, 0xf9, 0xee, 0x8c, 0x6a, 0x1f, 0x72,
+				0xa9, 0x58, 0xc4, 0xcc, 0xca, 0x41, 0x4c, 0xbf, 0x8b, 0x18, 0xf9, 0x53, 0xe, 0xb1, 0x40, 0x54,
 			},
 			squareSize: appconsts.MaxSquareSize,
 			shares:     generateShares(appconsts.MaxSquareSize*appconsts.MaxSquareSize, 99),
 		},
 	}
 
 	for _, tt := range tests {
-		tt := tt
-		eds, err := ExtendShares(tt.squareSize, tt.shares)
-		require.NoError(t, err)
-		resdah := NewDataAvailabilityHeader(eds)
-		require.Equal(t, tt.squareSize*2, uint64(len(resdah.ColumnRoots)), tt.name)
-		require.Equal(t, tt.squareSize*2, uint64(len(resdah.RowsRoots)), tt.name)
-		require.Equal(t, tt.expectedHash, resdah.hash, tt.name)
+		t.Run(tt.name, func(t *testing.T) {
+			eds, err := ExtendShares(tt.squareSize, tt.shares)
+			require.NoError(t, err)
+			resdah := NewDataAvailabilityHeader(eds)
+			require.Equal(t, tt.squareSize*2, uint64(len(resdah.ColumnRoots)), tt.name)
+			require.Equal(t, tt.squareSize*2, uint64(len(resdah.RowsRoots)), tt.name)
+			require.Equal(t, tt.expectedHash, resdah.hash, tt.name)
+		})
 	}
 }
 

pkg/prove/proof_test.go

@@ -152,14 +152,14 @@ func TestTxShareIndex(t *testing.T) {
 		{appconsts.FirstCompactShareContentSize + appconsts.ContinuationCompactShareContentSize + 1, 2},
 		{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 2), 2},
 		{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 2) + 1, 3},
-		// 81 compact shares + partially filled out last share
+		// 81 full compact shares then a partially filled out 82nd share (which is index 81 because 0-indexed)
 		{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 160, 81},
-		// 81 compact shares + full last share
-		{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 246, 81},
-		// 82 compact shares + one byte in last share
-		{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 247, 82},
-		// 82 compact shares + two bytes in last share
-		{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 248, 82},
+		// 81 full compact shares then a full 82nd share
+		{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 501, 81},
+		// 82 full compact shares then one byte in 83rd share
+		{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 502, 82},
+		// 82 compact shares then two bytes in 83rd share
+		{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 503, 82},
 	}
 
 	for _, tt := range tests {

pkg/shares/reserved_bytes.go

@@ -0,0 +1,39 @@
+package shares
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+
+	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+)
+
+// NewReservedBytes returns a byte slice of length
+// appconsts.CompactShareReservedBytes that contains a varint of the byteIndex
+// of the first unit that starts in a compact share. If no unit starts in the
+// compact share, ReservedBytes is [0, 0].
+func NewReservedBytes(byteIndex uint64) ([]byte, error) {
+	if byteIndex >= appconsts.ShareSize {
+		return []byte{}, fmt.Errorf("byte index %d must be less than share size %d", byteIndex, appconsts.ShareSize)
+	}
+	reservedBytes := make([]byte, appconsts.CompactShareReservedBytes)
+	binary.PutUvarint(reservedBytes, byteIndex)
+	return reservedBytes, nil
+}
+
+// ParseReservedBytes parses a byte slice of length
+// appconsts.CompactShareReservedBytes into a byteIndex.
+func ParseReservedBytes(reservedBytes []byte) (uint64, error) {
+	if len(reservedBytes) != appconsts.CompactShareReservedBytes {
+		return 0, fmt.Errorf("reserved bytes must be of length %d", appconsts.CompactShareReservedBytes)
+	}
+	reader := bytes.NewReader(reservedBytes)
+	byteIndex, err := binary.ReadUvarint(reader)
+	if err != nil {
+		return 0, err
+	}
+	if byteIndex >= appconsts.ShareSize {
+		return 0, fmt.Errorf("reserved bytes varint %d must be less than share size %d", byteIndex, appconsts.ShareSize)
+	}
+	return byteIndex, nil
+}

pkg/shares/reserved_bytes_test.go

@@ -0,0 +1,84 @@
+package shares
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseReservedBytes(t *testing.T) {
+	type testCase struct {
+		name      string
+		input     []byte
+		want      uint64
+		expectErr bool
+	}
+	testCases := []testCase{
+		{"byte index of 0", []byte{0, 0}, 0, false},
+		{"byte index of 2", []byte{2, 0}, 2, false},
+		{"byte index of 4", []byte{4, 0}, 4, false},
+		{"byte index of 8", []byte{8, 0}, 8, false},
+		{"byte index of 16", []byte{16, 0}, 16, false},
+		{"byte index of 32", []byte{32, 0}, 32, false},
+		{"byte index of 64", []byte{64, 0}, 64, false},
+		{"byte index of 128", []byte{128, 1}, 128, false},
+		{"byte index of 256", []byte{128, 2}, 256, false},
+		{"byte index of 511", []byte{255, 3}, 511, false},
+
+		// error cases
+		{"empty", []byte{}, 0, true},
+		{"too few reserved bytes", []byte{1}, 0, true},
+		{"too many reserved bytes", []byte{3, 3, 3}, 0, true},
+		{"byte index of 512 is equal to share size", []byte{128, 4}, 0, true},
+		{"byte index of 1000 is greater than share size", []byte{232, 7}, 0, true},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := ParseReservedBytes(tc.input)
+			if tc.expectErr {
+				assert.Error(t, err)
+				return
+			}
+			assert.NoError(t, err)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestNewReservedBytes(t *testing.T) {
+	type testCase struct {
+		name      string
+		input     uint64
+		want      []byte
+		expectErr bool
+	}
+	testCases := []testCase{
+		{"byte index of 0", 0, []byte{0, 0}, false},
+		{"byte index of 2", 2, []byte{2, 0}, false},
+		{"byte index of 4", 4, []byte{4, 0}, false},
+		{"byte index of 8", 8, []byte{8, 0}, false},
+		{"byte index of 16", 16, []byte{16, 0}, false},
+		{"byte index of 32", 32, []byte{32, 0}, false},
+		{"byte index of 64", 64, []byte{64, 0}, false},
+		{"byte index of 128", 128, []byte{128, 1}, false},
+		{"byte index of 256", 256, []byte{128, 2}, false},
+		{"byte index of 511", 511, []byte{255, 3}, false},
+
+		// error cases
+		{"byte index of 512 is equal to share size", 512, []byte{}, true},
+		{"byte index of 1000 is greater than share size", 1000, []byte{}, true},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := NewReservedBytes(tc.input)
+			if tc.expectErr {
+				assert.Error(t, err)
+				return
+			}
+			assert.NoError(t, err)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}