feat: support pigeon operator keys (#330)

# Related Github tickets

- VolumeFi/paloma#857
- VolumeFi/paloma#934
- palomachain/paloma#1025

# Background

In order to support multiple signing keys for Pigeons, we recently released a change for Paloma that introduced message metadata fields for all Paloma messages, allowing Pigeons to define signers other than the creator address.

This change includes:

- Removing lens as a dependency, replacing it with a slimmed down in-house solution called `ion`
- Support to define multiple signature keys in the pigeon config, see below for additional details
- Automatic key rotation when sending messages to Paloma
- Automatic metadata injection into messages
- Introducing `validator-key` as a configuration field for the paloma chain inside Pigeon. This used to be the `signing-key` field, which remains to be supported for now

The new signing keys are completely optional. In case there's no change to the configuration, Pigeon will read the old existing `signature-key` field and use it for both the message creator and signer when injecting metadata. See below on how to enable support for multiple signing keys.

Going forward, we'll be referring to the old `signing-key` as `validator-key`.

#### Support for multiple signing keys

By default, Pigeon will use your validator key to sign any transactions sent to Paloma. In high throughput environments, this may lead to `account sequence mismatch` errors.
It's possible to define more than one signing key to be used in rotation to combat this issue. In order to do so, you will need to create a number of new keys and register them with Pigeon like this:

```bash
# First, create a number of new keys. You may create an arbitrary amount of keys.
palomad keys add pigeon-operator-alpha
palomad keys add pigeon-operator-bravo
palomad keys add pigeon-operator-charlie

# Second, your new addresses will need to receive an active feegrant from your validator address.
# This step is very important. It's not enough to simply fund those addresses manually.
# The active feegrant is considered a "permission" to send transactions from your validator address".
palomad tx feegrant grant $VALIDATOR_ADDRESS pigeon-operator-alpha --fees 500ugrain -y
palomad tx feegrant grant $VALIDATOR_ADDRESS pigeon-operator-bravo --fees 500ugrain -y
palomad tx feegrant grant $VALIDATOR_ADDRESS pigeon-operator-charlie --fees 500ugrain -y
```

After creating your signing keys, all you need to do is register them with Pigeon by adding the following to your pigeon config. Make sure to restart the service after making these changes.

```yaml
paloma:
  signing-keys:
    - pigeon-operator-alpha
    - pigeon-operator-bravo
    - pigeon-operator-charlie
```

# Testing completed

- [x] test coverage exists or has been added/updated
- [x] tested in a private testnet

# Breaking changes

- [x] I have checked my code for breaking changes

---

* feat: support multiple signing keys with rotation

* fix: use master address when sending messages

* fix: dead lock on mutex acquisition

* feat: add fee grant

* chore: add logging

* feat: inject message metadata

* fix: retrieve signer address for metadata

* fix: remove superfluous logging

* feat: remove locking from paloma calls

* fix: use paloma wrapper constructor

* chore: remove all lens dependencies

* fix: CI pipeline

* chore: move signature keys configuration to paloma only

* chore: update go.mod paloma reference

* doc: update README

Author: byte-bandit

.github/workflows/ci-test.yml

@@ -4,25 +4,38 @@ on:
       - "*"
 
 jobs:
-  run-tests:
+  run_tests:
+    name: Running tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
+          cache: false
       - name: Run tests
         run: go test -v ./...
+  run_linters:
+    name: Running linters
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+          cache: false
       - name: Go lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.51.2
+          version: v1.55.2
           args: --verbose
           # Optional: if set to true then the all caching functionality will be complete disabled,
           #           takes precedence over all other caching options.
-          # skip-cache: true
+          skip-cache: true
 
           # Optional: if set to true then the action don't cache or restore ~/go/pkg.
           skip-pkg-cache: true

README.md

@@ -176,7 +176,7 @@ paloma:
   keyring-dir: ~/.paloma
   keyring-pass-env-name: PALOMA_KEYRING_PASS
   keyring-type: os
-  signing-key: ${VALIDATOR}
+  validator-key: ${VALIDATOR}
   base-rpc-url: http://localhost:26657
   gas-adjustment: 3.0
   gas-prices: 0.01ugrain
@@ -256,6 +256,34 @@ evm:
     tx-type: 2
 ```
 
+#### Support for multiple signing keys
+
+By default, Pigeon will use your validator key to sign any transactions sent to Paloma. In high throughput environments, this may lead to `account sequence mismatch` errors.
+It's possible to define more than one signing key to be used in rotation to combat this issue. In order to do so, you will need to create a number of new keys and register them with Pigeon like this:
+
+```bash
+# First, create a number of new keys. You may create an arbitrary amount of keys.
+palomad keys add pigeon-operator-alpha
+palomad keys add pigeon-operator-bravo
+palomad keys add pigeon-operator-charlie
+
+# Second, your new addresses will need to receive an active feegrant from your validator address.
+# This step is very important. It's not enough to simply fund those addresses manually.
+# The active feegrant is considered a "permission" to send transactions from your validator address".
+palomad tx feegrant grant $VALIDATOR_ADDRESS pigeon-operator-alpha --fees 500ugrain -y
+palomad tx feegrant grant $VALIDATOR_ADDRESS pigeon-operator-bravo --fees 500ugrain -y
+palomad tx feegrant grant $VALIDATOR_ADDRESS pigeon-operator-charlie --fees 500ugrain -y
+```
+
+After creating your signing keys, all you need to do is register them with Pigeon by adding the following to your pigeon config. Make sure to restart the service after making these changes.
+
+```yaml
+paloma:
+  signing-keys:
+    - pigeon-operator-alpha
+    - pigeon-operator-bravo
+    - pigeon-operator-charlie
+```
 
 ### Start pigeon
 

app/app.go

@@ -14,16 +14,16 @@ import (
 	gravitytypes "github.com/palomachain/paloma/x/gravity/types"
 	palomatypes "github.com/palomachain/paloma/x/paloma/types"
 	valsettypes "github.com/palomachain/paloma/x/valset/types"
-	"github.com/palomachain/pigeon/chain"
 	"github.com/palomachain/pigeon/chain/evm"
 	"github.com/palomachain/pigeon/chain/paloma"
 	"github.com/palomachain/pigeon/config"
 	"github.com/palomachain/pigeon/health"
 	"github.com/palomachain/pigeon/relayer"
+	"github.com/palomachain/pigeon/util/ion"
+	"github.com/palomachain/pigeon/util/ion/byop"
+	"github.com/palomachain/pigeon/util/rotator"
 	"github.com/palomachain/pigeon/util/time"
 	log "github.com/sirupsen/logrus"
-	"github.com/strangelove-ventures/lens/byop"
-	lens "github.com/strangelove-ventures/lens/client"
 )
 
 const (
@@ -64,7 +64,7 @@ func Relayer() *relayer.Relayer {
 	if _relayer == nil {
 		_relayer = relayer.New(
 			Config(),
-			*PalomaClient(),
+			PalomaClient(),
 			EvmFactory(),
 			Time(),
 			relayer.Config{
@@ -118,6 +118,16 @@ func Config() *config.Config {
 				"err": err,
 			}).Fatal("couldn't read config file")
 		}
+
+		if len(cnf.Paloma.ValidatorKey) < 1 {
+			// TODO: Remove legacy SigningKey field after successful migration
+			cnf.Paloma.ValidatorKey = cnf.Paloma.SigningKey
+		}
+		if len(cnf.Paloma.SigningKeys) < 1 {
+			log.Info("No signing key collection provided, falling back to using validator key for signing")
+			cnf.Paloma.SigningKeys = []string{cnf.Paloma.ValidatorKey}
+		}
+
 		_config = cnf
 	}
 
@@ -127,26 +137,29 @@ func Config() *config.Config {
 func PalomaClient() *paloma.Client {
 	if _palomaClient == nil {
 		palomaConfig := Config().Paloma
-
-		lensConfig := palomaLensClientConfig(palomaConfig)
+		clientCfg := palomaClientConfig(palomaConfig)
 
 		// HACK: \n is added at the end of a password because github.com/cosmos/[email protected]/client/input/input.go at line 93 would return an EOF error which then would fail
 		// Should be fixed with https://github.com/cosmos/cosmos-sdk/pull/11796
 		passInput := strings.NewReader(config.KeyringPassword(palomaConfig.KeyringPassEnvName) + "\n")
-
-		lensClient := whoops.Must(chain.NewChainClient(
-			lensConfig,
+		ionClient := whoops.Must(ion.NewClient(
+			clientCfg,
 			passInput,
 			os.Stdout,
 		))
 
-		_palomaClient = &paloma.Client{
-			L:             lensClient,
-			GRPCClient:    paloma.GRPCClientDowner{W: lensClient},
-			MessageSender: paloma.MessageSenderDowner{W: lensClient},
-			PalomaConfig:  palomaConfig,
+		// Always pass configurations as pointers to enable updates during runtime!
+		// TODO: Move rotator & paloma client wrapper functionalities directly into ion client
+		fn := func(s string) {
+			clientCfg.Key = s
 		}
-		_palomaClient.Init()
+		r := rotator.New(fn, palomaConfig.SigningKeys...)
+
+		grpcWrapper := &paloma.GRPCClientWrapper{W: ionClient}
+		senderWrapper := paloma.NewPalomaMessageSender(r, ionClient)
+		_palomaClient = paloma.NewClient(palomaConfig, grpcWrapper, ionClient, senderWrapper, ionClient)
+		senderWrapper.WithCreatorProvider(_palomaClient.GetCreator)
+		senderWrapper.WithSignerProvider(_palomaClient.GetSigner)
 	}
 	return _palomaClient
 }
@@ -159,8 +172,8 @@ func defaultValue[T comparable](proposedVal T, defaultVal T) T {
 	return proposedVal
 }
 
-func palomaLensClientConfig(palomaConfig config.Paloma) *lens.ChainClientConfig {
-	modules := lens.ModuleBasics[:]
+func palomaClientConfig(palomaConfig config.Paloma) *ion.ChainClientConfig {
+	modules := ion.ModuleBasics[:]
 
 	modules = append(modules, byop.Module{
 		ModuleName: "paloma",
@@ -172,7 +185,6 @@ func palomaLensClientConfig(palomaConfig config.Paloma) *lens.ChainClientConfig
 					&consensustypes.MsgAddMessagesSignatures{},
 					&valsettypes.MsgAddExternalChainInfoForValidator{},
 					&valsettypes.MsgKeepAlive{},
-					&consensustypes.MsgDeleteJob{},
 					&consensustypes.MsgAddEvidence{},
 					&consensustypes.MsgSetPublicAccessData{},
 					&consensustypes.MsgSetErrorData{},
@@ -210,8 +222,8 @@ func palomaLensClientConfig(palomaConfig config.Paloma) *lens.ChainClientConfig
 		},
 	})
 
-	return &lens.ChainClientConfig{
-		Key:            palomaConfig.SigningKey,
+	return &ion.ChainClientConfig{
+		Key:            palomaConfig.SigningKeys[0],
 		ChainID:        defaultValue(palomaConfig.ChainID, "paloma"),
 		RPCAddr:        defaultValue(palomaConfig.BaseRPCURL, "http://127.0.0.1:26657"),
 		AccountPrefix:  defaultValue(palomaConfig.AccountPrefix, "paloma"),