From ee4983c6c78eb2fede23594ac0f422dd4d1cd30b Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Tue, 25 Jul 2023 13:06:33 -0700 Subject: [PATCH 1/2] export client trusting period --- relayer/processor/message_processor.go | 1 + relayer/processor/metrics.go | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/relayer/processor/message_processor.go b/relayer/processor/message_processor.go index fe355cfa7..1e742dd81 100644 --- a/relayer/processor/message_processor.go +++ b/relayer/processor/message_processor.go @@ -144,6 +144,7 @@ func (mp *messageProcessor) shouldUpdateClientNow(ctx context.Context, src, dst if mp.metrics != nil { timeToExpiration := dst.clientState.TrustingPeriod - time.Since(consensusHeightTime) mp.metrics.SetClientExpiration(src.info.PathName, dst.info.ChainID, dst.clientState.ClientID, fmt.Sprint(dst.clientState.TrustingPeriod.String()), timeToExpiration) + mp.metrics.SetClientTrustingPeriod(src.info.PathName, dst.info.ChainID, dst.info.ClientID, time.Duration(dst.clientState.TrustingPeriod)) } if shouldUpdateClientNow { diff --git a/relayer/processor/metrics.go b/relayer/processor/metrics.go index 96de766b6..d1dcdaf74 100644 --- a/relayer/processor/metrics.go +++ b/relayer/processor/metrics.go @@ -15,6 +15,7 @@ type PrometheusMetrics struct { WalletBalance *prometheus.GaugeVec FeesSpent *prometheus.GaugeVec ClientExpiration *prometheus.GaugeVec + ClientTrustingPeriod *prometheus.GaugeVec } func (m *PrometheusMetrics) AddPacketsObserved(path, chain, channel, port, eventType string, count int) { @@ -41,11 +42,16 @@ func (m *PrometheusMetrics) SetClientExpiration(pathName, chain, clientID, trust m.ClientExpiration.WithLabelValues(pathName, chain, clientID, trustingPeriod).Set(timeToExpiration.Seconds()) } +func (m *PrometheusMetrics) SetClientTrustingPeriod(pathName, chain, clientID string, trustingPeriod time.Duration) { + m.ClientTrustingPeriod.WithLabelValues(pathName, chain, clientID).Set(trustingPeriod.Abs().Seconds()) +} + func NewPrometheusMetrics() *PrometheusMetrics { packetLabels := []string{"path", "chain", "channel", "port", "type"} heightLabels := []string{"chain"} walletLabels := []string{"chain", "key", "address", "denom"} clientExpirationLables := []string{"path_name", "chain", "client_id", "trusting_period"} + clientTrustingPeriodLables := []string{"path_name", "chain", "client_id"} registry := prometheus.NewRegistry() registerer := promauto.With(registry) return &PrometheusMetrics{ @@ -74,5 +80,9 @@ func NewPrometheusMetrics() *PrometheusMetrics { Name: "cosmos_relayer_client_expiration_seconds", Help: "Seconds until the client expires", }, clientExpirationLables), + ClientTrustingPeriod: registerer.NewGaugeVec(prometheus.GaugeOpts{ + Name: "cosmos_relayer_client_trusting_period_seconds", + Help: "The trusting period (in seconds) of the client", + }, clientTrustingPeriodLables), } } From 041e64e48f1e0db3778a6f574e089bd8159b3700 Mon Sep 17 00:00:00 2001 From: Dan Kanefsky Date: Tue, 25 Jul 2023 13:30:01 -0700 Subject: [PATCH 2/2] update docs --- docs/advanced_usage.md | 40 ++++++++++++------------------------ relayer/processor/metrics.go | 2 +- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/docs/advanced_usage.md b/docs/advanced_usage.md index a0b518b07..1b6c37595 100644 --- a/docs/advanced_usage.md +++ b/docs/advanced_usage.md @@ -10,33 +10,19 @@ you can use `http://$IP:5183/relayer/metrics` as a target for your prometheus sc Exported metrics: -| **Exported Metric** | **Description** | **Type** | -|:----------------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:--------:| -| cosmos_relayer_observed_packets | The total number of observed packets | Counter | -| cosmos_relayer_relayed_packets | The total number of relayed packets | Counter | -| cosmos_relayer_chain_latest_height | The current height of the chain | Gauge | -| cosmos_relayer_wallet_balance | The current balance for the relayer's wallet | Gauge | -| cosmos_relayer_fees_spent | The amount of fees spent from the relayer's wallet | Gauge | -| cosmos_relayer_tx_failure |
The total number of tx failures broken up into catagories .
Categories:
- "packet messages are redundant"
- "insufficient funds"
- "invalid coins"
- "out of gas"
- "incorrect account sequence"

"Tx Failure" is the the catch all bucket| Counter | -**Example metrics** - -``` -go_goroutines 29 -... -go_threads 39 -... -observed_packets{chain="cosmoshub-4",channel="channel-141",path="hubosmo",port="transfer",type="acknowledge_packet"} 57 -observed_packets{chain="cosmoshub-4",channel="channel-141",path="hubosmo",port="transfer",type="recv_packet"} 103 -observed_packets{chain="cosmoshub-4",channel="channel-141",path="hubosmo",port="transfer",type="send_packet"} 58 -observed_packets{chain="osmosis-1",channel="channel-0",path="hubosmo",port="transfer",type="acknowledge_packet"} 107 -observed_packets{chain="osmosis-1",channel="channel-0",path="hubosmo",port="transfer",type="recv_packet"} 60 -observed_packets{chain="osmosis-1",channel="channel-0",path="hubosmo",port="transfer",type="send_packet"} 102 -... -relayed_packets{chain="cosmoshub-4",channel="channel-141",path="hubosmo",port="transfer",type="acknowledge_packet"} 31 -relayed_packets{chain="cosmoshub-4",channel="channel-141",path="hubosmo",port="transfer",type="recv_packet"} 65 -relayed_packets{chain="osmosis-1",channel="channel-0",path="hubosmo",port="transfer",type="acknowledge_packet"} 36 -relayed_packets{chain="osmosis-1",channel="channel-0",path="hubosmo",port="transfer",type="recv_packet"} 35 -``` +| **Exported Metric** | **Description** | **Type** | +|:---------------------------------------------: |:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |:--------: | +| cosmos_relayer_observed_packets | The total number of observed packets | Counter | +| cosmos_relayer_relayed_packets | The total number of relayed packets | Counter | +| cosmos_relayer_chain_latest_height | The current height of the chain | Gauge | +| cosmos_relayer_wallet_balance | The current balance for the relayer's wallet | Gauge | +| cosmos_relayer_fees_spent | The amount of fees spent from the relayer's wallet | Gauge | +| cosmos_relayer_tx_failure |
The total number of tx failures broken up into categories:
- "packet messages are redundant"
- "insufficient funds"
- "invalid coins"
- "out of gas"


"Tx Failure" is the the catch all bucket | Counter | +| cosmos_relayer_block_query_errors_total | The total number of block query failures. The failures are separated into two categories:
- "RPC Client"
- "IBC Header" | Counter | +| cosmos_relayer_client_expiration_seconds | Seconds until the client expires | Gauge | +| cosmos_relayer_client_trusting_period_seconds | The trusting period (in seconds) of the client | Gauge | + + --- diff --git a/relayer/processor/metrics.go b/relayer/processor/metrics.go index c9546713d..0b40e3d1a 100644 --- a/relayer/processor/metrics.go +++ b/relayer/processor/metrics.go @@ -94,7 +94,7 @@ func NewPrometheusMetrics() *PrometheusMetrics { }, txFailureLabels), BlockQueryFailure: registerer.NewCounterVec(prometheus.CounterOpts{ Name: "cosmos_relayer_block_query_errors_total", - Help: "The total number of block query failures. The failures are separated into two catagories: 'RPC Client' and 'IBC Header'", + Help: "The total number of block query failures. The failures are separated into two categories: 'RPC Client' and 'IBC Header'", }, blockQueryFailureLabels), ClientExpiration: registerer.NewGaugeVec(prometheus.GaugeOpts{ Name: "cosmos_relayer_client_expiration_seconds",