From 041364d38c34807fb6eb2f767548f9a44ae7b028 Mon Sep 17 00:00:00 2001 From: David Herberth Date: Wed, 3 Apr 2024 09:24:20 +0200 Subject: [PATCH 1/4] wip --- requirements-base.txt | 2 +- requirements-dev-frozen.txt | 2 +- requirements-frozen.txt | 2 +- src/sentry/options/defaults.py | 9 ++++++++ src/sentry/relay/config/__init__.py | 17 ++++++++++++++ .../False/REGION.pysnap | 7 ++++++ .../True/REGION.pysnap | 7 ++++++ tests/sentry/relay/test_config.py | 22 +++++++++++++++++++ 8 files changed, 65 insertions(+), 3 deletions(-) diff --git a/requirements-base.txt b/requirements-base.txt index 04deebce004cb2..6d4ea4605e13c0 100644 --- a/requirements-base.txt +++ b/requirements-base.txt @@ -65,7 +65,7 @@ sentry-arroyo>=2.16.5 sentry-kafka-schemas>=0.1.65 sentry-ophio==0.2.6 sentry-redis-tools>=0.1.7 -sentry-relay>=0.8.55 +sentry-relay>=0.8.56 sentry-sdk>=1.44.0 snuba-sdk>=2.0.31 simplejson>=3.17.6 diff --git a/requirements-dev-frozen.txt b/requirements-dev-frozen.txt index 9467e856cfc525..1504b1655b86cf 100644 --- a/requirements-dev-frozen.txt +++ b/requirements-dev-frozen.txt @@ -179,7 +179,7 @@ sentry-forked-djangorestframework-stubs==3.14.5.post1 sentry-kafka-schemas==0.1.65 sentry-ophio==0.2.6 sentry-redis-tools==0.1.7 -sentry-relay==0.8.55 +sentry-relay==0.8.56 sentry-sdk==1.44.0 sentry-usage-accountant==0.0.10 simplejson==3.17.6 diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 5f9e37970f1c53..cc524d5c42eeaa 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -121,7 +121,7 @@ sentry-arroyo==2.16.5 sentry-kafka-schemas==0.1.65 sentry-ophio==0.2.6 sentry-redis-tools==0.1.7 -sentry-relay==0.8.55 +sentry-relay==0.8.56 sentry-sdk==1.44.0 sentry-usage-accountant==0.0.10 simplejson==3.17.6 diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index 01fcd7d692208f..b908d63a70b2bb 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -993,6 +993,15 @@ register( "relay.cardinality-limiter.error-sample-rate", default=0.01, flags=FLAG_AUTOMATOR_MODIFIABLE ) +# List of additional cardinality limits and selectors. +# +# ``` +# { +# "rollout_rate": 0.001, +# "limit": { .. Cardinality Limit .. } +# } +# ``` +register("relay.cardinality-limiter.limits", default=[], flags=FLAG_AUTOMATOR_MODIFIABLE) # Controls the encoding used in Relay for encoding distributions and sets # when writing to Kafka. diff --git a/src/sentry/relay/config/__init__.py b/src/sentry/relay/config/__init__.py index a9add44bb383cd..b5d9b22655bab6 100644 --- a/src/sentry/relay/config/__init__.py +++ b/src/sentry/relay/config/__init__.py @@ -248,6 +248,11 @@ class CardinalityLimit(TypedDict): namespace: str | None +class CardinalityLimitOption(TypedDict): + rollout_rate: NotRequired[float] + limit: CardinalityLimit + + def get_metrics_config(timeout: TimeChecker, project: Project) -> Mapping[str, Any] | None: metrics_config = {} @@ -280,6 +285,18 @@ def get_metrics_config(timeout: TimeChecker, project: Project) -> Mapping[str, A if id in passive_limits: limit["passive"] = True cardinality_limits.append(limit) + + clos: list[CardinalityLimitOption] = options.get("relay.cardinality-limiter.limits") + for clo in clos: + rollout_rate = clo.pop("rollout_rate", 1.0) + if (project.organization.id % 100000) / 100000 >= rollout_rate: + continue + + try: + cardinality_limits.append(clo["limit"]) + except KeyError: + pass + metrics_config["cardinalityLimits"] = cardinality_limits if features.has("organizations:metrics-blocking", project.organization): diff --git a/tests/sentry/relay/snapshots/test_config/test_project_config_cardinality_limits/False/REGION.pysnap b/tests/sentry/relay/snapshots/test_config/test_project_config_cardinality_limits/False/REGION.pysnap index 04710413fdabdf..81fc2a8e37a884 100644 --- a/tests/sentry/relay/snapshots/test_config/test_project_config_cardinality_limits/False/REGION.pysnap +++ b/tests/sentry/relay/snapshots/test_config/test_project_config_cardinality_limits/False/REGION.pysnap @@ -37,3 +37,10 @@ cardinalityLimits: window: granularitySeconds: 600 windowSeconds: 3600 +- id: test2 + limit: 80 + report: true + scope: name + window: + granularitySeconds: 800 + windowSeconds: 8000 diff --git a/tests/sentry/relay/snapshots/test_config/test_project_config_cardinality_limits/True/REGION.pysnap b/tests/sentry/relay/snapshots/test_config/test_project_config_cardinality_limits/True/REGION.pysnap index 7b94575c690a52..cf4b741c10a8b6 100644 --- a/tests/sentry/relay/snapshots/test_config/test_project_config_cardinality_limits/True/REGION.pysnap +++ b/tests/sentry/relay/snapshots/test_config/test_project_config_cardinality_limits/True/REGION.pysnap @@ -42,3 +42,10 @@ cardinalityLimits: window: granularitySeconds: 600 windowSeconds: 3600 +- id: test2 + limit: 80 + report: true + scope: name + window: + granularitySeconds: 800 + windowSeconds: 8000 diff --git a/tests/sentry/relay/test_config.py b/tests/sentry/relay/test_config.py index b63f32fae92446..56fcff62891734 100644 --- a/tests/sentry/relay/test_config.py +++ b/tests/sentry/relay/test_config.py @@ -957,6 +957,28 @@ def test_project_config_cardinality_limits(default_project, insta_snapshot, pass ] } + options["relay.cardinality-limiter.limits"] = [ + { + "rollout_rate": 0, + "limit": { + "id": "test1", + "window": {"windowSeconds": 7000, "granularitySeconds": 700}, + "limit": 70, + "scope": "name", + }, + }, + { + "rollout_rate": 1, + "limit": { + "id": "test2", + "window": {"windowSeconds": 8000, "granularitySeconds": 800}, + "limit": 80, + "scope": "name", + "report": True, + }, + }, + ] + features = Feature({"organizations:relay-cardinality-limiter": True}) with override_options(options), features: From ec602e2f51c7f301dbe9d35402c1b66b10ab7699 Mon Sep 17 00:00:00 2001 From: David Herberth Date: Wed, 3 Apr 2024 10:14:12 +0200 Subject: [PATCH 2/4] remove sessions drop --- src/sentry/relay/config/__init__.py | 3 --- tests/sentry/relay/test_config.py | 1 - 2 files changed, 4 deletions(-) diff --git a/src/sentry/relay/config/__init__.py b/src/sentry/relay/config/__init__.py index b5d9b22655bab6..8e0e1febf6fbe7 100644 --- a/src/sentry/relay/config/__init__.py +++ b/src/sentry/relay/config/__init__.py @@ -845,9 +845,6 @@ def _get_project_config( if _should_extract_abnormal_mechanism(project) else EXTRACT_METRICS_VERSION ), - "drop": features.has( - "organizations:release-health-drop-sessions", project.organization - ), } performance_score_profiles = [ diff --git a/tests/sentry/relay/test_config.py b/tests/sentry/relay/test_config.py index 56fcff62891734..79a3af15954c6a 100644 --- a/tests/sentry/relay/test_config.py +++ b/tests/sentry/relay/test_config.py @@ -497,7 +497,6 @@ def test_project_config_with_organizations_metrics_extraction( session_metrics = get_path(cfg, "config", "sessionMetrics") if has_metrics_extraction: assert session_metrics == { - "drop": False, "version": 2 if abnormal_mechanism_rollout else 1, } else: From b116deb16b358b3b4a6287accc67ed82e43ba972 Mon Sep 17 00:00:00 2001 From: David Herberth Date: Wed, 3 Apr 2024 11:35:07 +0200 Subject: [PATCH 3/4] pop -> get --- src/sentry/relay/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/relay/config/__init__.py b/src/sentry/relay/config/__init__.py index 8e0e1febf6fbe7..80f3b671f406d0 100644 --- a/src/sentry/relay/config/__init__.py +++ b/src/sentry/relay/config/__init__.py @@ -288,7 +288,7 @@ def get_metrics_config(timeout: TimeChecker, project: Project) -> Mapping[str, A clos: list[CardinalityLimitOption] = options.get("relay.cardinality-limiter.limits") for clo in clos: - rollout_rate = clo.pop("rollout_rate", 1.0) + rollout_rate = clo.get("rollout_rate", 1.0) if (project.organization.id % 100000) / 100000 >= rollout_rate: continue From 5906782eb0f2d940507fc653d812d6ddf3ee8876 Mon Sep 17 00:00:00 2001 From: David Herberth Date: Wed, 3 Apr 2024 11:46:41 +0200 Subject: [PATCH 4/4] more drops removed --- tests/sentry/api/endpoints/test_relay_projectconfigs_v2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/sentry/api/endpoints/test_relay_projectconfigs_v2.py b/tests/sentry/api/endpoints/test_relay_projectconfigs_v2.py index fb18da34da3793..b3f1c2a63250e8 100644 --- a/tests/sentry/api/endpoints/test_relay_projectconfigs_v2.py +++ b/tests/sentry/api/endpoints/test_relay_projectconfigs_v2.py @@ -357,7 +357,7 @@ def test_session_metrics_extraction(call_endpoint, task_runner, drop_sessions): for config in result["configs"].values(): config = config["config"] - assert config["sessionMetrics"] == {"version": 1, "drop": drop_sessions} + assert config["sessionMetrics"] == {"version": 1} @django_db_all @@ -378,5 +378,4 @@ def test_session_metrics_abnormal_mechanism_tag_extraction( config = config["config"] assert config["sessionMetrics"] == { "version": 2 if abnormal_mechanism_rollout else 1, - "drop": False, }