Skip to content

Commit 31fd99a

Browse files
tac0turtlemelekes
tac0turtle
and
melekes
authoredFeb 11, 2020
proto: add buf and protogen script (#4369)
* proto: add buf and protogen script - add buf with minimal changes - add protogen script to easier generate proto files Signed-off-by: Marko Baricevic <[email protected]> * add protoc needs * add some needed shell cmds * remove buf from tools as it is not needed everytime * add proto lint and breakage to ci * add section in changelog and upgrading files * address pr comments * remove space in circle config * remove spaces in makefile comment * add section on contributing on how to work with proto * bump buf to 0.7 * test bufbuild image * test install make in bufbuild image * revert to tendermintdev image * Update Makefile Co-Authored-By: Anton Kaliaev <[email protected]> Co-authored-by: Anton Kaliaev <[email protected]>
·
v2.0.0-tm-v0.34.20-rc1tendermint-v0.34.12-base
1 parent aeb6cc4 commit 31fd99a

File tree

17 files changed

+512
-324
lines changed

17 files changed

+512
-324
lines changed
 

‎.circleci/config.yml

+18
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ executors:
1414
- image: tendermintdev/docker-website-deployment
1515
environment:
1616
AWS_REGION: us-east-1
17+
protoc:
18+
docker:
19+
- image: tendermintdev/docker-protoc
1720

1821
commands:
1922
run_test:
@@ -72,6 +75,19 @@ jobs:
7275
root: "/tmp/bin"
7376
paths:
7477
- "."
78+
proto-lint:
79+
executor: protoc
80+
steps:
81+
- checkout
82+
- run:
83+
command: make proto-lint
84+
85+
proto-breakage:
86+
executor: protoc
87+
steps:
88+
- checkout
89+
- run:
90+
command: make proto-check-breaking
7591

7692
test_abci_apps:
7793
executor: golang
@@ -391,6 +407,8 @@ workflows:
391407
- test_abci_apps:
392408
requires:
393409
- setup_dependencies
410+
- proto-breakage
411+
- proto-lint
394412
- test_abci_cli:
395413
requires:
396414
- setup_dependencies

‎CHANGELOG_PENDING.md

+4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ program](https://hackerone.com/tendermint).
2222

2323
### IMPROVEMENTS:
2424

25+
- [proto] [\#4369] Add [buf](https://buf.build/) for usage with linting and checking if there are breaking changes with the master branch.
26+
- [proto] [\#4369] Add `make proto-gen` cmd to generate proto stubs outside of GOPATH.
27+
28+
2529
### BUG FIXES:
2630

2731
- [node] [#\4311] Use `GRPCMaxOpenConnections` when creating the gRPC server, not `MaxOpenConnections`

‎CONTRIBUTING.md

+4
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,10 @@ need. Instead of running `go get -u=patch`, which will update anything,
9898
specify exactly the dependency you want to update, eg.
9999
`GO111MODULE=on go get -u github.com/tendermint/go-amino@master`.
100100

101+
## Protobuf
102+
103+
When working with [protobuf](https://developers.google.com/protocol-buffers) there are a few things you should know. We use [buf](https://buf.build/) for our linting and breaking changes checking. If you would like to run linting and check if the changes you have made are breaking then you will have to install the needed dependencies with `make buf`. Then the linting cmd will be `make proto-lint` and the breaking changes check will be `make proto-check-breaking`. To generate new stubs based off of your changes you can run `make proto-gen` (you can do this outside of GOPATH).
104+
101105
## Vagrant
102106

103107
If you are a [Vagrant](https://www.vagrantup.com/) user, you can get started

‎Makefile

+41-36
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
PACKAGES=$(shell go list ./...)
22
OUTPUT?=build/tendermint
33

4-
INCLUDE = -I=${GOPATH}/src/github.com/tendermint/tendermint -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf
54
BUILD_TAGS?='tendermint'
65
LD_FLAGS = -X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD` -s -w
76
BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)"
@@ -12,8 +11,9 @@ all: check build test install
1211
include tools.mk
1312
include tests.mk
1413

15-
########################################
16-
### Build Tendermint
14+
###############################################################################
15+
### Build Tendermint ###
16+
###############################################################################
1717

1818
build:
1919
CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o $(OUTPUT) ./cmd/tendermint/
@@ -30,35 +30,41 @@ install:
3030
install_c:
3131
CGO_ENABLED=1 go install $(BUILD_FLAGS) -tags "$(BUILD_TAGS) cleveldb" ./cmd/tendermint
3232

33-
########################################
34-
### Protobuf
33+
###############################################################################
34+
### Protobuf ###
35+
###############################################################################
3536

36-
protoc_all: protoc_libs protoc_merkle protoc_abci protoc_grpc protoc_proto3types
37+
proto-all: proto-gen proto-lint proto-check-breaking
3738

38-
%.pb.go: %.proto
39+
proto-gen:
3940
## If you get the following error,
4041
## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory"
4142
## See https://stackoverflow.com/a/25518702
4243
## Note the $< here is substituted for the %.proto
4344
## Note the $@ here is substituted for the %.pb.go
44-
protoc $(INCLUDE) $< --gogo_out=Mgoogle/protobuf/timestamp.proto=github.com/golang/protobuf/ptypes/timestamp,Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration,plugins=grpc:../../..
45+
@sh scripts/protocgen.sh
4546

46-
########################################
47-
### Build ABCI
47+
proto-lint:
48+
@buf check lint --error-format=json
4849

49-
# see protobuf section above
50-
protoc_abci: abci/types/types.pb.go
50+
proto-check-breaking:
51+
@buf check breaking --against-input '.git#branch=master'
5152

52-
protoc_proto3types: types/proto3/block.pb.go
53+
.PHONY: proto-all proto-gen proto-lint proto-check-breaking
54+
55+
###############################################################################
56+
### Build ABCI ###
57+
###############################################################################
5358

5459
build_abci:
5560
@go build -mod=readonly -i ./abci/cmd/...
5661

5762
install_abci:
5863
@go install -mod=readonly ./abci/cmd/...
5964

60-
########################################
61-
### Distribution
65+
###############################################################################
66+
### Distribution ###
67+
###############################################################################
6268

6369
# dist builds binaries for all platforms and packages them for distribution
6470
# TODO add abci to these scripts
@@ -86,10 +92,9 @@ get_deps_bin_size:
8692
@find $(WORK) -type f -name "*.a" | xargs -I{} du -hxs "{}" | sort -rh | sed -e s:${WORK}/::g > deps_bin_size.log
8793
@echo "Results can be found here: $(CURDIR)/deps_bin_size.log"
8894

89-
########################################
90-
### Libs
91-
92-
protoc_libs: libs/kv/types.pb.go
95+
###############################################################################
96+
### Libs ###
97+
###############################################################################
9398

9499
# generates certificates for TLS testing in remotedb and RPC server
95100
gen_certs: clean_certs
@@ -105,13 +110,9 @@ clean_certs:
105110
rm -f rpc/lib/server/test.crt
106111
rm -f rpc/lib/server/test.key
107112

108-
protoc_grpc: rpc/grpc/types.pb.go
109-
110-
protoc_merkle: crypto/merkle/merkle.pb.go
111-
112-
113-
########################################
114-
### Formatting, linting, and vetting
113+
###############################################################################
114+
### Formatting, linting, and vetting ###
115+
###############################################################################
115116

116117
fmt:
117118
@go fmt ./...
@@ -122,8 +123,9 @@ lint:
122123

123124
DESTINATION = ./index.html.md
124125

125-
###########################################################
126-
### Documentation
126+
###############################################################################
127+
### Documentation ###
128+
###############################################################################
127129

128130
build-docs:
129131
cd docs && \
@@ -142,16 +144,18 @@ sync-docs:
142144
aws cloudfront create-invalidation --distribution-id ${CF_DISTRIBUTION_ID} --profile terraform --path "/*" ;
143145
.PHONY: sync-docs
144146

145-
###########################################################
146-
### Docker image
147+
###############################################################################
148+
### Docker image ###
149+
###############################################################################
147150

148151
build-docker:
149152
cp $(OUTPUT) DOCKER/tendermint
150153
docker build --label=tendermint --tag="tendermint/tendermint" DOCKER
151154
rm -rf DOCKER/tendermint
152155

153-
###########################################################
154-
### Local testnet using docker
156+
###############################################################################
157+
### Local testnet using docker ###
158+
###############################################################################
155159

156160
# Build linux binary on other platforms
157161
build-linux: tools
@@ -176,8 +180,9 @@ localnet-start: localnet-stop build-docker-localnode
176180
localnet-stop:
177181
docker-compose down
178182

179-
###########################################################
180-
### Remote full-nodes (sentry) using terraform and ansible
183+
###############################################################################
184+
### Remote full-nodes (sentry) using terraform and ansible ###
185+
###############################################################################
181186

182187
# Server management
183188
sentry-start:
@@ -216,7 +221,7 @@ contract-tests:
216221
# unless there is a reason not to.
217222
# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
218223
.PHONY: check build build_race build_abci dist install install_abci check_tools tools update_tools draw_deps \
219-
protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver fmt build-linux localnet-start \
220-
localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop protoc_grpc protoc_all \
224+
gen_certs clean_certs grpc_dbserver fmt build-linux localnet-start \
225+
localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop \
221226
build_c install_c test_with_deadlock cleanup_after_test_with_deadlock lint build-contract-tests-hooks contract-tests \
222227
build_c-amazonlinux

‎UPGRADING.md

+8
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,14 @@
33
This guide provides steps to be followed when you upgrade your applications to
44
a newer version of Tendermint Core.
55

6+
## Unreleased
7+
8+
<Overview>
9+
10+
### Protobuf Changes
11+
12+
When upgrading to version <version #> you will have to fetch the `third_party` directory along with the updated proto files.
13+
614
## v0.33.0
715

816
This release is not compatible with previous blockchains due to commit becoming signatures only and fields in the header have been removed.

‎abci/types/types.pb.go

+149-149
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎abci/types/types.proto

+3-3
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ option go_package = "github.com/tendermint/tendermint/abci/types";
44

55
// For more information on gogo.proto, see:
66
// https://github.com/gogo/protobuf/blob/master/extensions.md
7-
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
8-
import "github.com/tendermint/tendermint/crypto/merkle/merkle.proto";
9-
import "github.com/tendermint/tendermint/libs/kv/types.proto";
7+
import "third_party/proto/gogoproto/gogo.proto";
8+
import "crypto/merkle/merkle.proto";
9+
import "libs/kv/types.proto";
1010
import "google/protobuf/timestamp.proto";
1111
import "google/protobuf/duration.proto";
1212

‎buf.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
build:
2+
roots:
3+
- .
4+
lint:
5+
use:
6+
- MINIMAL
7+
- FILE_LOWER_SNAKE_CASE
8+
- UNARY_RPC
9+
except:
10+
- PACKAGE_DIRECTORY_MATCH
11+
ignore:
12+
- third_party
13+
breaking:
14+
use:
15+
- FILE
16+
- PACKAGE

‎crypto/merkle/merkle.pb.go

+29-55
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crypto/merkle/merkle.proto

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ option go_package = "github.com/tendermint/tendermint/crypto/merkle";
44

55
// For more information on gogo.proto, see:
66
// https://github.com/gogo/protobuf/blob/master/extensions.md
7-
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
7+
import "third_party/proto/gogoproto/gogo.proto";
88

99
option (gogoproto.marshaler_all) = true;
1010
option (gogoproto.unmarshaler_all) = true;

‎libs/kv/types.pb.go

+11-11
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎libs/kv/types.proto

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ syntax = "proto3";
22
package tendermint.libs.kv;
33
option go_package = "github.com/tendermint/tendermint/libs/kv";
44

5-
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
5+
import "third_party/proto/gogoproto/gogo.proto";
66

77
option (gogoproto.marshaler_all) = true;
88
option (gogoproto.unmarshaler_all) = true;

‎rpc/grpc/types.pb.go

+39-65
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎rpc/grpc/types.proto

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
syntax = "proto3";
22
package tendermint.rpc.grpc;
3-
option go_package = "github.com/tendermint/tendermint/rpc/grpc;core_grpc";
3+
option go_package = "github.com/tendermint/tendermint/rpc/grpc;coregrpc";
44

5-
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
6-
import "github.com/tendermint/tendermint/abci/types/types.proto";
5+
import "third_party/proto/gogoproto/gogo.proto";
6+
import "abci/types/types.proto";
77

88
option (gogoproto.marshaler_all) = true;
99
option (gogoproto.unmarshaler_all) = true;

‎scripts/protocgen.sh

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env bash
2+
3+
set -eo pipefail
4+
5+
proto_dirs=$(find . -path ./third_party -prune -o -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq)
6+
for dir in $proto_dirs; do
7+
protoc \
8+
-I. \
9+
--gogo_out=Mgoogle/protobuf/timestamp.proto=github.com/golang/protobuf/ptypes/timestamp,Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration,plugins=grpc,paths=source_relative:. \
10+
$(find "${dir}" -name '*.proto')
11+
done

‎third_party/proto/gogoproto/gogo.proto

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
// Protocol Buffers for Go with Gadgets
2+
//
3+
// Copied from https://github.com/gogo/protobuf/blob/master/gogoproto/gogo.proto
4+
//
5+
// Copyright (c) 2013, The GoGo Authors. All rights reserved.
6+
// http://github.com/gogo/protobuf
7+
//
8+
// Redistribution and use in source and binary forms, with or without
9+
// modification, are permitted provided that the following conditions are
10+
// met:
11+
//
12+
// * Redistributions of source code must retain the above copyright
13+
// notice, this list of conditions and the following disclaimer.
14+
// * Redistributions in binary form must reproduce the above
15+
// copyright notice, this list of conditions and the following disclaimer
16+
// in the documentation and/or other materials provided with the
17+
// distribution.
18+
//
19+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20+
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21+
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22+
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23+
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24+
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25+
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26+
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27+
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28+
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29+
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30+
31+
syntax = "proto2";
32+
package gogoproto;
33+
34+
import "google/protobuf/descriptor.proto";
35+
36+
option java_package = "com.google.protobuf";
37+
option java_outer_classname = "GoGoProtos";
38+
option go_package = "github.com/gogo/protobuf/gogoproto";
39+
40+
extend google.protobuf.EnumOptions {
41+
optional bool goproto_enum_prefix = 62001;
42+
optional bool goproto_enum_stringer = 62021;
43+
optional bool enum_stringer = 62022;
44+
optional string enum_customname = 62023;
45+
optional bool enumdecl = 62024;
46+
}
47+
48+
extend google.protobuf.EnumValueOptions {
49+
optional string enumvalue_customname = 66001;
50+
}
51+
52+
extend google.protobuf.FileOptions {
53+
optional bool goproto_getters_all = 63001;
54+
optional bool goproto_enum_prefix_all = 63002;
55+
optional bool goproto_stringer_all = 63003;
56+
optional bool verbose_equal_all = 63004;
57+
optional bool face_all = 63005;
58+
optional bool gostring_all = 63006;
59+
optional bool populate_all = 63007;
60+
optional bool stringer_all = 63008;
61+
optional bool onlyone_all = 63009;
62+
63+
optional bool equal_all = 63013;
64+
optional bool description_all = 63014;
65+
optional bool testgen_all = 63015;
66+
optional bool benchgen_all = 63016;
67+
optional bool marshaler_all = 63017;
68+
optional bool unmarshaler_all = 63018;
69+
optional bool stable_marshaler_all = 63019;
70+
71+
optional bool sizer_all = 63020;
72+
73+
optional bool goproto_enum_stringer_all = 63021;
74+
optional bool enum_stringer_all = 63022;
75+
76+
optional bool unsafe_marshaler_all = 63023;
77+
optional bool unsafe_unmarshaler_all = 63024;
78+
79+
optional bool goproto_extensions_map_all = 63025;
80+
optional bool goproto_unrecognized_all = 63026;
81+
optional bool gogoproto_import = 63027;
82+
optional bool protosizer_all = 63028;
83+
optional bool compare_all = 63029;
84+
optional bool typedecl_all = 63030;
85+
optional bool enumdecl_all = 63031;
86+
87+
optional bool goproto_registration = 63032;
88+
optional bool messagename_all = 63033;
89+
90+
optional bool goproto_sizecache_all = 63034;
91+
optional bool goproto_unkeyed_all = 63035;
92+
}
93+
94+
extend google.protobuf.MessageOptions {
95+
optional bool goproto_getters = 64001;
96+
optional bool goproto_stringer = 64003;
97+
optional bool verbose_equal = 64004;
98+
optional bool face = 64005;
99+
optional bool gostring = 64006;
100+
optional bool populate = 64007;
101+
optional bool stringer = 67008;
102+
optional bool onlyone = 64009;
103+
104+
optional bool equal = 64013;
105+
optional bool description = 64014;
106+
optional bool testgen = 64015;
107+
optional bool benchgen = 64016;
108+
optional bool marshaler = 64017;
109+
optional bool unmarshaler = 64018;
110+
optional bool stable_marshaler = 64019;
111+
112+
optional bool sizer = 64020;
113+
114+
optional bool unsafe_marshaler = 64023;
115+
optional bool unsafe_unmarshaler = 64024;
116+
117+
optional bool goproto_extensions_map = 64025;
118+
optional bool goproto_unrecognized = 64026;
119+
120+
optional bool protosizer = 64028;
121+
optional bool compare = 64029;
122+
123+
optional bool typedecl = 64030;
124+
125+
optional bool messagename = 64033;
126+
127+
optional bool goproto_sizecache = 64034;
128+
optional bool goproto_unkeyed = 64035;
129+
}
130+
131+
extend google.protobuf.FieldOptions {
132+
optional bool nullable = 65001;
133+
optional bool embed = 65002;
134+
optional string customtype = 65003;
135+
optional string customname = 65004;
136+
optional string jsontag = 65005;
137+
optional string moretags = 65006;
138+
optional string casttype = 65007;
139+
optional string castkey = 65008;
140+
optional string castvalue = 65009;
141+
142+
optional bool stdtime = 65010;
143+
optional bool stdduration = 65011;
144+
optional bool wktpointer = 65012;
145+
146+
optional string castrepeated = 65013;
147+
}

‎tools.mk

+27
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,14 @@ mkfile_dir := $(shell cd $(shell dirname $(mkfile_path)); pwd)
3838
# Go tools
3939
###
4040

41+
BIN ?= /usr/local/bin
42+
UNAME_S ?= $(shell uname -s)
43+
UNAME_M ?= $(shell uname -m)
44+
4145
TOOLS_DESTDIR ?= $(GOPATH)/bin
4246

47+
BUF_VERSION ?= 0.7.0
48+
4349
CERTSTRAP = $(TOOLS_DESTDIR)/certstrap
4450
PROTOBUF = $(TOOLS_DESTDIR)/protoc
4551
GOODMAN = $(TOOLS_DESTDIR)/goodman
@@ -65,6 +71,27 @@ $(PROTOBUF):
6571
@echo "Get GoGo Protobuf"
6672
@go get github.com/gogo/protobuf/protoc-gen-gogo@v1.3.1
6773

74+
buf: protoc-gen-buf-check-breaking protoc-gen-buf-check-lint
75+
@echo "Installing buf..."
76+
@curl -sSL \
77+
"https://github.com/bufbuild/buf/releases/download/v${BUF_VERSION}/buf-${UNAME_S}-${UNAME_M}" \
78+
-o "${BIN}/buf" && \
79+
chmod +x "${BIN}/buf"
80+
81+
protoc-gen-buf-check-breaking:
82+
@echo "Installing protoc-gen-buf-check-breaking..."
83+
@curl -sSL \
84+
"https://github.com/bufbuild/buf/releases/download/v${BUF_VERSION}/protoc-gen-buf-check-breaking-${UNAME_S}-${UNAME_M}" \
85+
-o "${BIN}/protoc-gen-buf-check-breaking" && \
86+
chmod +x "${BIN}/protoc-gen-buf-check-breaking"
87+
88+
protoc-gen-buf-check-lint:
89+
@echo "Installing protoc-gen-buf-check-lint..."
90+
@curl -sSL \
91+
"https://github.com/bufbuild/buf/releases/download/v${BUF_VERSION}/protoc-gen-buf-check-lint-${UNAME_S}-${UNAME_M}" \
92+
-o "${BIN}/protoc-gen-buf-check-lint" && \
93+
chmod +x "${BIN}/protoc-gen-buf-check-lint"
94+
6895
goodman: $(GOODMAN)
6996
$(GOODMAN):
7097
@echo "Get Goodman"

0 commit comments

Comments
 (0)
Please sign in to comment.