From 9344f18cba9d8dc7fe7c71e7f0517c403eb4c882 Mon Sep 17 00:00:00 2001 From: Ryan Stewart Date: Tue, 20 Sep 2022 16:17:27 -0500 Subject: [PATCH] query for 1 minute of annotations instead of 1 day When I ramped the load test up on staging, it started making query nodes run out of memory. I traced it back to the annotations queries here, which appears to run a query similar to the others, but it fails to account for the fact that "annotations" are just little json blobs to keep track of discrete events. They don't form any time series and aren't rolled up. It's fine to query for a day's metrics because you'll get downsampled data, but annotations just accumulate, and if you run the load test for the whole day, by the end of it, there are far too many annotations to fit into memory when generating the payload. --- scripts/query.py | 7 +++++-- scripts/tests.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/query.py b/scripts/query.py index 3ef9118..0700050 100644 --- a/scripts/query.py +++ b/scripts/query.py @@ -14,7 +14,8 @@ class AbstractQueryGenerator(AbstractGenerator): - one_day = (1000 * 60 * 60 * 24) + one_minute = (1000 * 60) + one_day = (one_minute * 60 * 24) query_interval_name = None @@ -106,7 +107,9 @@ def make_request(self, logger, time, tenant_id=None): if tenant_id is None: tenant_id = self.user.get_tenant_id() to = time - frm = time - self.one_day + # Unlike time series data, annotations don't have any kind of rollup, so querying for a day's data can return a + # huge payload, causing memory issues on the servers when you generate a lot of load. + frm = time - self.one_minute url = "%s/v2.0/%s/events/getEvents?from=%d&until=%d" % ( self.config['query_url'], tenant_id, frm, to) result = self.request.GET(url) diff --git a/scripts/tests.py b/scripts/tests.py index c5e92ee..996df6d 100644 --- a/scripts/tests.py +++ b/scripts/tests.py @@ -633,7 +633,7 @@ def test_query_make_AnnotationsQuery_request(self): response = qq.make_request(None, 1000, 30) self.assertEqual(req.get_url, "http://metrics.example.org/v2.0/30/events/" + - "getEvents?from=-86399000&until=1000") + "getEvents?from=-59000&until=1000") self.assertIs(req, response.request)