Skip to content

Commit e970884

Browse files
authored
Dimensional Metrics (#815)
* Wiring dimensional metrics * Squashed commit of the following: commit c2d4629 Author: Timothy Pansino <[email protected]> Date: Wed May 10 15:59:13 2023 -0700 Add required option for tox v4 (#795) * Add required option for tox v4 * Update tox in GHA * Remove py27 no-cache-dir commit a963649 Author: Hannah Stepanek <[email protected]> Date: Tue May 9 10:46:39 2023 -0700 Run coverage around pytest (#813) * Run coverage around pytest * Trigger tests * Fixup * Add redis client_no_touch to ignore list * Temporarily remove kafka from coverage * Remove coverage for old libs commit 3d82845 Author: Lalleh Rafeei <[email protected]> Date: Wed May 3 14:50:30 2023 -0700 Omit some frameworks from coverage analysis (#810) * Omit some frameworks from coverage analysis * Remove commas * Change format of omit * Add relative_files option to coverage * Add absolute directory * Add envsitepackagedir * Add coveragerc file * Add codecov.yml * [Mega-Linter] Apply linters fixes * Revert coveragerc file settings * Add files in packages and more frameworks * Remove commented line --------- Co-authored-by: lrafeei <[email protected]> Co-authored-by: Hannah Stepanek <[email protected]> commit fd0fa35 Author: Uma Annamalai <[email protected]> Date: Tue May 2 10:55:36 2023 -0700 Add testing for genshi and mako. (#799) * Add testing for genshi and mako. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: umaannamalai <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit be4fb3d Author: Lalleh Rafeei <[email protected]> Date: Mon May 1 16:01:09 2023 -0700 Add tests for Waitress (#797) * Change import format * Initial commit * Add more tests to adapter_waitress * Remove commented out code * [Mega-Linter] Apply linters fixes * Add assertions to all tests * Add more NR testing to waitress --------- Co-authored-by: lrafeei <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 7103506 Author: Hannah Stepanek <[email protected]> Date: Mon May 1 14:12:31 2023 -0700 Add tests for pyodbc (#796) * Add tests for pyodbc * Move imports into tests to get import coverage * Fixup: remove time import * Trigger tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 363122a Author: Hannah Stepanek <[email protected]> Date: Mon May 1 13:34:35 2023 -0700 Pin virtualenv, fix pip arg deprecation & disable kafka tests (#803) * Pin virtualenv * Fixup: use 20.21.1 instead * Replace install-options with config-settings See pypa/pip#11358. * Temporarily disable kafka tests * Add dimensional stats table to stats engine * Add attribute processing to metric identity * Add testing for dimensional metrics * Cover tags as list not dict * Commit suggestions from code review
1 parent a726625 commit e970884

13 files changed

+583
-6
lines changed

newrelic/api/application.py

+8
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,14 @@ def record_custom_metrics(self, metrics):
142142
if self.active and metrics:
143143
self._agent.record_custom_metrics(self._name, metrics)
144144

145+
def record_dimensional_metric(self, name, value, tags=None):
146+
if self.active:
147+
self._agent.record_dimensional_metric(self._name, name, value, tags)
148+
149+
def record_dimensional_metrics(self, metrics):
150+
if self.active and metrics:
151+
self._agent.record_dimensional_metrics(self._name, metrics)
152+
145153
def record_custom_event(self, event_type, params):
146154
if self.active:
147155
self._agent.record_custom_event(self._name, event_type, params)

newrelic/api/transaction.py

+51-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@
6868
from newrelic.core.custom_event import create_custom_event
6969
from newrelic.core.log_event_node import LogEventNode
7070
from newrelic.core.stack_trace import exception_stack
71-
from newrelic.core.stats_engine import CustomMetrics, SampledDataSet
71+
from newrelic.core.stats_engine import CustomMetrics, DimensionalMetrics, SampledDataSet
7272
from newrelic.core.thread_utilization import utilization_tracker
7373
from newrelic.core.trace_cache import (
7474
TraceCacheActiveTraceError,
@@ -309,6 +309,7 @@ def __init__(self, application, enabled=None, source=None):
309309
self.synthetics_header = None
310310

311311
self._custom_metrics = CustomMetrics()
312+
self._dimensional_metrics = DimensionalMetrics()
312313

313314
global_settings = application.global_settings
314315

@@ -591,6 +592,7 @@ def __exit__(self, exc, value, tb):
591592
apdex_t=self.apdex,
592593
suppress_apdex=self.suppress_apdex,
593594
custom_metrics=self._custom_metrics,
595+
dimensional_metrics=self._dimensional_metrics,
594596
guid=self.guid,
595597
cpu_time=self._cpu_user_time_value,
596598
suppress_transaction_trace=self.suppress_transaction_trace,
@@ -1597,6 +1599,16 @@ def record_custom_metrics(self, metrics):
15971599
for name, value in metrics:
15981600
self._custom_metrics.record_custom_metric(name, value)
15991601

1602+
def record_dimensional_metric(self, name, value, tags=None):
1603+
self._dimensional_metrics.record_dimensional_metric(name, value, tags)
1604+
1605+
def record_dimensional_metrics(self, metrics):
1606+
for metric in metrics:
1607+
name, value = metric[:2]
1608+
tags = metric[2] if len(metric) >= 3 else None
1609+
1610+
self._dimensional_metrics.record_dimensional_metric(name, value, tags)
1611+
16001612
def record_custom_event(self, event_type, params):
16011613
settings = self._settings
16021614

@@ -1908,6 +1920,44 @@ def record_custom_metrics(metrics, application=None):
19081920
application.record_custom_metrics(metrics)
19091921

19101922

1923+
def record_dimensional_metric(name, value, tags=None, application=None):
1924+
if application is None:
1925+
transaction = current_transaction()
1926+
if transaction:
1927+
transaction.record_dimensional_metric(name, value, tags)
1928+
else:
1929+
_logger.debug(
1930+
"record_dimensional_metric has been called but no "
1931+
"transaction was running. As a result, the following metric "
1932+
"has not been recorded. Name: %r Value: %r Tags: %r. To correct this "
1933+
"problem, supply an application object as a parameter to this "
1934+
"record_dimensional_metrics call.",
1935+
name,
1936+
value,
1937+
tags,
1938+
)
1939+
elif application.enabled:
1940+
application.record_dimensional_metric(name, value, tags)
1941+
1942+
1943+
def record_dimensional_metrics(metrics, application=None):
1944+
if application is None:
1945+
transaction = current_transaction()
1946+
if transaction:
1947+
transaction.record_dimensional_metrics(metrics)
1948+
else:
1949+
_logger.debug(
1950+
"record_dimensional_metrics has been called but no "
1951+
"transaction was running. As a result, the following metrics "
1952+
"have not been recorded: %r. To correct this problem, "
1953+
"supply an application object as a parameter to this "
1954+
"record_dimensional_metric call.",
1955+
list(metrics),
1956+
)
1957+
elif application.enabled:
1958+
application.record_dimensional_metrics(metrics)
1959+
1960+
19111961
def record_custom_event(event_type, params, application=None):
19121962
"""Record a custom event.
19131963

newrelic/common/metric_utils.py

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""
16+
This module implements functions for creating a unique identity from a name and set of tags for use in dimensional metrics.
17+
"""
18+
19+
from newrelic.core.attribute import process_user_attribute
20+
21+
22+
def create_metric_identity(name, tags=None):
23+
if tags:
24+
# Convert dicts to an iterable of tuples, other iterables should already be in this form
25+
if isinstance(tags, dict):
26+
tags = tags.items()
27+
28+
# Apply attribute system sanitization.
29+
# process_user_attribute returns (None, None) for results that fail sanitization.
30+
# The filter removes these results from the iterable before creating the frozenset.
31+
tags = frozenset(filter(lambda args: args[0] is not None, map(lambda args: process_user_attribute(*args), tags)))
32+
33+
tags = tags or None # Set empty iterables after filtering to None
34+
35+
return (name, tags)

newrelic/core/agent.py

+27
Original file line numberDiff line numberDiff line change
@@ -524,6 +524,33 @@ def record_custom_metrics(self, app_name, metrics):
524524

525525
application.record_custom_metrics(metrics)
526526

527+
def record_dimensional_metric(self, app_name, name, value, tags=None):
528+
"""Records a basic metric for the named application. If there has
529+
been no prior request to activate the application, the metric is
530+
discarded.
531+
532+
"""
533+
534+
application = self._applications.get(app_name, None)
535+
if application is None or not application.active:
536+
return
537+
538+
application.record_dimensional_metric(name, value, tags)
539+
540+
def record_dimensional_metrics(self, app_name, metrics):
541+
"""Records the metrics for the named application. If there has
542+
been no prior request to activate the application, the metric is
543+
discarded. The metrics should be an iterable yielding tuples
544+
consisting of the name and value.
545+
546+
"""
547+
548+
application = self._applications.get(app_name, None)
549+
if application is None or not application.active:
550+
return
551+
552+
application.record_dimensional_metrics(metrics)
553+
527554
def record_custom_event(self, app_name, event_type, params):
528555
application = self._applications.get(app_name, None)
529556
if application is None or not application.active:

newrelic/core/application.py

+50
Original file line numberDiff line numberDiff line change
@@ -510,6 +510,9 @@ def connect_to_data_collector(self, activate_agent):
510510
with self._stats_custom_lock:
511511
self._stats_custom_engine.reset_stats(configuration)
512512

513+
with self._stats_lock:
514+
self._stats_engine.reset_stats(configuration)
515+
513516
# Record an initial start time for the reporting period and
514517
# clear record of last transaction processed.
515518

@@ -860,6 +863,50 @@ def record_custom_metrics(self, metrics):
860863
self._global_events_account += 1
861864
self._stats_custom_engine.record_custom_metric(name, value)
862865

866+
def record_dimensional_metric(self, name, value, tags=None):
867+
"""Record a dimensional metric against the application independent
868+
of a specific transaction.
869+
870+
NOTE that this will require locking of the stats engine for
871+
dimensional metrics and so under heavy use will have performance
872+
issues. It is better to record the dimensional metric against an
873+
active transaction as they will then be aggregated at the end of
874+
the transaction when all other metrics are aggregated and so no
875+
additional locking will be required.
876+
877+
"""
878+
879+
if not self._active_session:
880+
return
881+
882+
with self._stats_lock:
883+
self._global_events_account += 1
884+
self._stats_engine.record_dimensional_metric(name, value, tags)
885+
886+
def record_dimensional_metrics(self, metrics):
887+
"""Record a set of dimensional metrics against the application
888+
independent of a specific transaction.
889+
890+
NOTE that this will require locking of the stats engine for
891+
dimensional metrics and so under heavy use will have performance
892+
issues. It is better to record the dimensional metric against an
893+
active transaction as they will then be aggregated at the end of
894+
the transaction when all other metrics are aggregated and so no
895+
additional locking will be required.
896+
897+
"""
898+
899+
if not self._active_session:
900+
return
901+
902+
with self._stats_lock:
903+
for metric in metrics:
904+
name, value = metric[:2]
905+
tags = metric[2] if len(metric) >= 3 else None
906+
907+
self._global_events_account += 1
908+
self._stats_engine.record_dimensional_metric(name, value, tags)
909+
863910
def record_custom_event(self, event_type, params):
864911
if not self._active_session:
865912
return
@@ -1452,11 +1499,14 @@ def harvest(self, shutdown=False, flexible=False):
14521499
_logger.debug("Normalizing metrics for harvest of %r.", self._app_name)
14531500

14541501
metric_data = stats.metric_data(metric_normalizer)
1502+
dimensional_metric_data = stats.dimensional_metric_data(metric_normalizer)
14551503

14561504
_logger.debug("Sending metric data for harvest of %r.", self._app_name)
14571505

14581506
# Send metrics
14591507
self._active_session.send_metric_data(self._period_start, period_end, metric_data)
1508+
if dimensional_metric_data:
1509+
self._active_session.send_dimensional_metric_data(self._period_start, period_end, dimensional_metric_data)
14601510

14611511
_logger.debug("Done sending data for harvest of %r.", self._app_name)
14621512

newrelic/core/data_collector.py

+21
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131

3232
_logger = logging.getLogger(__name__)
3333

34+
DIMENSIONAL_METRIC_DATA_TEMP = [] # TODO: REMOVE THIS
35+
3436

3537
class Session(object):
3638
PROTOCOL = AgentProtocol
@@ -135,6 +137,25 @@ def send_metric_data(self, start_time, end_time, metric_data):
135137
payload = (self.agent_run_id, start_time, end_time, metric_data)
136138
return self._protocol.send("metric_data", payload)
137139

140+
def send_dimensional_metric_data(self, start_time, end_time, metric_data):
141+
"""Called to submit dimensional metric data for specified period of time.
142+
Time values are seconds since UNIX epoch as returned by the
143+
time.time() function. The metric data should be iterable of
144+
specific metrics.
145+
146+
NOTE: This data is sent not sent to the normal agent endpoints but is sent
147+
to the MELT API endpoints to keep the entity separate. This is for use
148+
with the machine learning integration only.
149+
"""
150+
151+
payload = (self.agent_run_id, start_time, end_time, metric_data)
152+
# return self._protocol.send("metric_data", payload)
153+
154+
# TODO: REMOVE THIS. Replace with actual protocol.
155+
DIMENSIONAL_METRIC_DATA_TEMP.append(payload)
156+
_logger.debug("Dimensional Metrics: %r" % metric_data)
157+
return 200
158+
138159
def send_log_events(self, sampling_info, log_event_data):
139160
"""Called to submit sample set for log events."""
140161

0 commit comments

Comments
 (0)