Skip to content

Commit 83bc051

Browse files
ViBiOhViBiOh
authored
feat(aws): Move tags enablement environment variable to settings (#1004)
Signed-off-by: Vincent Boutour <vincent.boutour@datadoghq.com>
1 parent 389b911 commit 83bc051

File tree

7 files changed

+92
-32
lines changed

7 files changed

+92
-32
lines changed

‎aws/logs_monitoring/caching/cloudwatch_log_group_cache.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
DD_S3_CACHE_DIRNAME,
1414
DD_S3_LOG_GROUP_CACHE_DIRNAME,
1515
DD_TAGS_CACHE_TTL_SECONDS,
16+
get_fetch_log_group_tags,
1617
)
1718
from telemetry import send_forwarder_internal_metrics
1819

@@ -61,7 +62,7 @@ def get(self, log_group_arn):
6162
return self._fetch_log_group_tags(log_group_arn)
6263

6364
def _should_fetch_tags(self):
64-
return os.environ.get("DD_FETCH_LOG_GROUP_TAGS", "false").lower() == "true"
65+
return get_fetch_log_group_tags()
6566

6667
def _fetch_log_group_tags(self, log_group_arn):
6768
# first, check in-memory cache

‎aws/logs_monitoring/caching/lambda_cache.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
import os
2-
31
from botocore.exceptions import ClientError
42

53
from caching.base_tags_cache import BaseTagsCache
@@ -8,6 +6,7 @@
86
DD_S3_LAMBDA_CACHE_FILENAME,
97
DD_S3_LAMBDA_CACHE_LOCK_FILENAME,
108
GET_RESOURCES_LAMBDA_FILTER,
9+
get_fetch_lambda_tags,
1110
)
1211
from telemetry import send_forwarder_internal_metrics
1312

@@ -19,7 +18,7 @@ def __init__(self, prefix):
1918
)
2019

2120
def should_fetch_tags(self):
22-
return os.environ.get("DD_FETCH_LAMBDA_TAGS", "false").lower() == "true"
21+
return get_fetch_lambda_tags()
2322

2423
def build_tags_cache(self):
2524
"""Makes API calls to GetResources to get the live tags of the account's Lambda functions

‎aws/logs_monitoring/caching/s3_tags_cache.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
1-
import os
21
from botocore.exceptions import ClientError
2+
33
from caching.base_tags_cache import BaseTagsCache
44
from caching.common import parse_get_resources_response_for_tags_by_arn
5-
from telemetry import send_forwarder_internal_metrics
65
from settings import (
76
DD_S3_TAGS_CACHE_FILENAME,
87
DD_S3_TAGS_CACHE_LOCK_FILENAME,
98
GET_RESOURCES_S3_FILTER,
9+
get_fetch_s3_tags,
1010
)
11+
from telemetry import send_forwarder_internal_metrics
1112

1213

1314
class S3TagsCache(BaseTagsCache):
@@ -18,7 +19,7 @@ def __init__(self, prefix):
1819

1920
def should_fetch_tags(self):
2021
# set it to true if we don't have the environment variable set to keep the default behavior
21-
return os.environ.get("DD_FETCH_S3_TAGS", "true").lower() == "true"
22+
return get_fetch_s3_tags()
2223

2324
def build_tags_cache(self):
2425
"""Makes API calls to GetResources to get the live tags of the account's S3 buckets

‎aws/logs_monitoring/caching/step_functions_cache.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
1-
import os
21
from botocore.exceptions import ClientError
2+
33
from caching.base_tags_cache import BaseTagsCache
44
from caching.common import (
5-
sanitize_aws_tag_string,
65
parse_get_resources_response_for_tags_by_arn,
6+
sanitize_aws_tag_string,
77
)
8-
from telemetry import send_forwarder_internal_metrics
98
from settings import (
109
DD_S3_STEP_FUNCTIONS_CACHE_FILENAME,
1110
DD_S3_STEP_FUNCTIONS_CACHE_LOCK_FILENAME,
1211
GET_RESOURCES_STEP_FUNCTIONS_FILTER,
12+
get_fetch_step_functions_tags,
1313
)
14+
from telemetry import send_forwarder_internal_metrics
1415

1516

1617
class StepFunctionsTagsCache(BaseTagsCache):
@@ -22,7 +23,7 @@ def __init__(self, prefix):
2223
)
2324

2425
def should_fetch_tags(self):
25-
return os.environ.get("DD_FETCH_STEP_FUNCTIONS_TAGS", "false").lower() == "true"
26+
return get_fetch_step_functions_tags()
2627

2728
def build_tags_cache(self):
2829
"""Makes API calls to GetResources to get the live tags of the account's Step Functions

‎aws/logs_monitoring/settings.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,37 @@ def __init__(self, name, pattern, placeholder, enabled=True):
271271
"DD_MULTILINE_LOG_REGEX_PATTERN", default=None
272272
)
273273

274+
DD_FETCH_S3_TAGS = get_env_var("DD_FETCH_S3_TAGS", default="true", boolean=True)
275+
276+
DD_FETCH_LOG_GROUP_TAGS = get_env_var(
277+
"DD_FETCH_LOG_GROUP_TAGS", default="true", boolean=True
278+
)
279+
280+
DD_FETCH_LAMBDA_TAGS = get_env_var(
281+
"DD_FETCH_LAMBDA_TAGS", default="false", boolean=True
282+
)
283+
284+
DD_FETCH_STEP_FUNCTIONS_TAGS = get_env_var(
285+
"DD_FETCH_STEP_FUNCTIONS_TAGS", default="false", boolean=True
286+
)
287+
288+
289+
def get_fetch_s3_tags():
290+
return DD_FETCH_S3_TAGS
291+
292+
293+
def get_fetch_log_group_tags():
294+
return DD_FETCH_LOG_GROUP_TAGS
295+
296+
297+
def get_fetch_lambda_tags():
298+
return DD_FETCH_LAMBDA_TAGS
299+
300+
301+
def get_fetch_step_functions_tags():
302+
return DD_FETCH_STEP_FUNCTIONS_TAGS
303+
304+
274305
DD_SOURCE = "ddsource"
275306
DD_CUSTOM_TAGS = "ddtags"
276307
DD_SERVICE = "service"

‎aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py‎

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,34 @@
11
import json
2-
import unittest
32
import os
3+
import sys
4+
import unittest
5+
from importlib import reload
46
from time import time
5-
from unittest.mock import patch, MagicMock
67
from unittest import mock
8+
from unittest.mock import MagicMock, patch
9+
710
from approvaltests.approvals import verify_as_json
811

12+
from caching.lambda_cache import LambdaTagsCache
913
from enhanced_lambda_metrics import (
10-
parse_metrics_from_report_log,
11-
parse_metrics_from_json_report_log,
12-
parse_lambda_tags_from_arn,
13-
generate_enhanced_lambda_metrics,
1414
create_out_of_memory_enhanced_metric,
15+
generate_enhanced_lambda_metrics,
16+
parse_lambda_tags_from_arn,
17+
parse_metrics_from_json_report_log,
18+
parse_metrics_from_report_log,
1519
)
1620

17-
from caching.lambda_cache import LambdaTagsCache
18-
1921

2022
class TestEnhancedLambdaMetrics(unittest.TestCase):
2123
maxDiff = None
2224
malformed_report = "REPORT invalid report log line"
2325
standard_report = (
24-
"REPORT RequestId: 8edab1f8-7d34-4a8e-a965-15ccbbb78d4c "
25-
"Duration: 0.62 ms Billed Duration: 100 ms Memory Size: 128 MB Max Memory Used: 51 MB"
26+
"REPORT RequestId: 8edab1f8-7d34-4a8e-a965-15ccbbb78d4c "
27+
"Duration: 0.62 ms Billed Duration: 100 ms Memory Size: 128 MB Max Memory Used: 51 MB"
2628
)
2729
cold_start_report = (
28-
"REPORT RequestId: 8edab1f8-7d34-4a8e-a965-15ccbbb78d4c "
29-
"Duration: 0.81 ms Billed Duration: 100 ms Memory Size: 128 MB Max Memory Used: 90 MB Init Duration: 1234 ms"
30+
"REPORT RequestId: 8edab1f8-7d34-4a8e-a965-15ccbbb78d4c "
31+
"Duration: 0.81 ms Billed Duration: 100 ms Memory Size: 128 MB Max Memory Used: 90 MB Init Duration: 1234 ms"
3032
)
3133
report_with_xray = (
3234
"REPORT RequestId: 814ba7cb-071e-4181-9a09-fa41db5bccad\tDuration: 1711.87 ms\t"
@@ -303,7 +305,10 @@ def test_generate_enhanced_lambda_metrics_once_with_missing_arn(self):
303305

304306
@patch("caching.base_tags_cache.send_forwarder_internal_metrics")
305307
@patch("caching.lambda_cache.send_forwarder_internal_metrics")
308+
@patch.dict(os.environ, {"DD_FETCH_LAMBDA_TAGS": "true"})
306309
def test_generate_enhanced_lambda_metrics_refresh_s3_cache(self, mock1, mock2):
310+
reload(sys.modules["settings"])
311+
307312
tags_cache = LambdaTagsCache("")
308313
tags_cache.get_cache_from_s3 = MagicMock(return_value=({}, 1000))
309314
tags_cache.acquire_s3_cache_lock = MagicMock()
@@ -334,13 +339,12 @@ def test_generate_enhanced_lambda_metrics_refresh_s3_cache(self, mock1, mock2):
334339
"timestamp": 10000,
335340
}
336341

337-
os.environ["DD_FETCH_LAMBDA_TAGS"] = "True"
338342
generate_enhanced_lambda_metrics(logs_input, tags_cache)
339343
tags_cache.get_cache_from_s3.assert_called_once()
340344
tags_cache.build_tags_cache.assert_called_once()
341345
tags_cache.write_cache_to_s3.assert_called_once()
342-
del os.environ["DD_FETCH_LAMBDA_TAGS"]
343346

347+
@patch.dict(os.environ, {"DD_FETCH_LAMBDA_TAGS": "true"})
344348
@patch("caching.lambda_cache.LambdaTagsCache.release_s3_cache_lock")
345349
@patch("caching.lambda_cache.LambdaTagsCache.acquire_s3_cache_lock")
346350
@patch("caching.lambda_cache.LambdaTagsCache.get_resources_paginator")
@@ -358,6 +362,8 @@ def test_generate_enhanced_lambda_metrics_client_error(
358362
mock_acquire_lock,
359363
mock_release_lock,
360364
):
365+
reload(sys.modules["settings"])
366+
361367
mock_acquire_lock.return_value = True
362368
mock_get_s3_cache.return_value = (
363369
{},
@@ -391,22 +397,23 @@ def test_generate_enhanced_lambda_metrics_client_error(
391397
"timestamp": 10000,
392398
}
393399

394-
os.environ["DD_FETCH_LAMBDA_TAGS"] = "True"
395400
generate_enhanced_lambda_metrics(logs_input, tags_cache)
396401
mock_get_s3_cache.assert_called_once()
397402
mock_get_s3_cache.reset_mock()
398403
mock_get_resources_paginator.assert_called_once()
399404
paginator.paginate.assert_called_once()
400405
assert mock_base_tags_cache_forward_metrics.call_count == 1
401406
assert mock_lambda_cache_forward_metrics.call_count == 2
402-
del os.environ["DD_FETCH_LAMBDA_TAGS"]
403407

404408
@patch("caching.lambda_cache.send_forwarder_internal_metrics")
405409
@patch("caching.base_tags_cache.send_forwarder_internal_metrics")
406410
@patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3")
411+
@patch.dict(os.environ, {"DD_FETCH_LAMBDA_TAGS": "true"})
407412
def test_generate_enhanced_lambda_metrics_timeout(
408413
self, mock_get_s3_cache, mock_forward_metrics, mock_base_forward_metrics
409414
):
415+
reload(sys.modules["settings"])
416+
410417
mock_get_s3_cache.return_value = (
411418
{
412419
"arn:aws:lambda:us-east-1:0:function:cloudwatch-event": [
@@ -437,17 +444,18 @@ def test_generate_enhanced_lambda_metrics_timeout(
437444
"timestamp": 1591714946151,
438445
}
439446

440-
os.environ["DD_FETCH_LAMBDA_TAGS"] = "True"
441447
generated_metrics = generate_enhanced_lambda_metrics(logs_input, tags_cache)
442448
verify_as_json(generated_metrics)
443-
del os.environ["DD_FETCH_LAMBDA_TAGS"]
444449

445450
@patch("caching.lambda_cache.send_forwarder_internal_metrics")
446451
@patch("telemetry.send_forwarder_internal_metrics")
447452
@patch("caching.lambda_cache.LambdaTagsCache.get_cache_from_s3")
453+
@patch.dict(os.environ, {"DD_FETCH_LAMBDA_TAGS": "true"})
448454
def test_generate_enhanced_lambda_metrics_out_of_memory(
449455
self, mock_get_s3_cache, mock_forward_metrics, mock_base_forward_metrics
450456
):
457+
reload(sys.modules["settings"])
458+
451459
mock_get_s3_cache.return_value = (
452460
{
453461
"arn:aws:lambda:us-east-1:0:function:cloudwatch-event": [
@@ -478,10 +486,8 @@ def test_generate_enhanced_lambda_metrics_out_of_memory(
478486
"timestamp": 1591714946151,
479487
}
480488

481-
os.environ["DD_FETCH_LAMBDA_TAGS"] = "True"
482489
generated_metrics = generate_enhanced_lambda_metrics(logs_input, tags_cache)
483490
verify_as_json(generated_metrics)
484-
del os.environ["DD_FETCH_LAMBDA_TAGS"]
485491

486492

487493
if __name__ == "__main__":

‎aws/logs_monitoring/tests/test_enrichment.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import json
2+
import os
3+
import sys
24
import unittest
3-
from unittest.mock import MagicMock
5+
from importlib import reload
6+
from unittest.mock import MagicMock, patch
47

58
from approvaltests.approvals import verify_as_json
69

@@ -181,7 +184,10 @@ def test_lambda_event_bad_arn(self):
181184
add_metadata_to_lambda_log(event, cache_layer)
182185
verify_as_json(event)
183186

187+
@patch.dict(os.environ, {"DD_FETCH_LAMBDA_TAGS": "false"})
184188
def test_lambda_event_wo_service(self):
189+
reload(sys.modules["settings"])
190+
185191
cache_layer = CacheLayer("")
186192
event = {
187193
"lambda": {
@@ -191,7 +197,10 @@ def test_lambda_event_wo_service(self):
191197
add_metadata_to_lambda_log(event, cache_layer)
192198
verify_as_json(event)
193199

200+
@patch.dict(os.environ, {"DD_FETCH_LAMBDA_TAGS": "true"})
194201
def test_lambda_event_w_custom_tags_wo_service(self):
202+
reload(sys.modules["settings"])
203+
195204
cache_layer = CacheLayer("")
196205
cache_layer._lambda_cache.get = MagicMock(
197206
return_value=["service:customtags_service"]
@@ -204,7 +213,10 @@ def test_lambda_event_w_custom_tags_wo_service(self):
204213
add_metadata_to_lambda_log(event, cache_layer)
205214
verify_as_json(event)
206215

216+
@patch.dict(os.environ, {"DD_FETCH_LAMBDA_TAGS": "true"})
207217
def test_lambda_event_w_custom_tags_w_service(self):
218+
reload(sys.modules["settings"])
219+
208220
cache_layer = CacheLayer("")
209221
cache_layer._lambda_cache.get = MagicMock(
210222
return_value=["service:customtags_service"]
@@ -218,7 +230,10 @@ def test_lambda_event_w_custom_tags_w_service(self):
218230
add_metadata_to_lambda_log(event, cache_layer)
219231
verify_as_json(event)
220232

233+
@patch.dict(os.environ, {"DD_FETCH_LAMBDA_TAGS": "false"})
221234
def test_lambda_event_w_service(self):
235+
reload(sys.modules["settings"])
236+
222237
cache_layer = CacheLayer("")
223238
event = {
224239
"lambda": {
@@ -229,7 +244,10 @@ def test_lambda_event_w_service(self):
229244
add_metadata_to_lambda_log(event, cache_layer)
230245
verify_as_json(event)
231246

247+
@patch.dict(os.environ, {"DD_FETCH_LAMBDA_TAGS": "false"})
232248
def test_lambda_event_w_service_and_ddtags(self):
249+
reload(sys.modules["settings"])
250+
233251
cache_layer = CacheLayer("")
234252
event = {
235253
"lambda": {
@@ -241,7 +259,10 @@ def test_lambda_event_w_service_and_ddtags(self):
241259
add_metadata_to_lambda_log(event, cache_layer)
242260
verify_as_json(event)
243261

262+
@patch.dict(os.environ, {"DD_FETCH_LAMBDA_TAGS": "true"})
244263
def test_lambda_event_w_custom_tags_env(self):
264+
reload(sys.modules["settings"])
265+
245266
cache_layer = CacheLayer("")
246267
cache_layer._lambda_cache.get = MagicMock(return_value=["env:customtags_env"])
247268
event = {

0 commit comments

Comments
 (0)