Skip to content
This repository was archived by the owner on Jun 13, 2023. It is now read-only.

Commit b5a56c3

Browse files
authored
feat(urrlib3): collecting payload (#356)
1 parent 45270fb commit b5a56c3

5 files changed

Lines changed: 115 additions & 9 deletions

File tree

epsagon/events/urllib3.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ def __init__(self, wrapped, instance, args, kwargs, start_time, response,
7676
add_data_if_needed(
7777
self.resource['metadata'],
7878
'request_headers',
79-
dict(headers)
79+
dict(headers) if headers else {}
8080
)
8181

8282
add_data_if_needed(
@@ -98,7 +98,7 @@ def update_response(self, response):
9898
:return: None
9999
"""
100100
self.resource['metadata']['status_code'] = response.status
101-
headers = dict(response.getheaders())
101+
headers = dict(response.getheaders()) if response.getheaders() else {}
102102
self.resource = update_http_headers(
103103
self.resource,
104104
headers
@@ -113,7 +113,13 @@ def update_response(self, response):
113113
'response_headers',
114114
headers
115115
)
116-
response_body = response.peek()
116+
response_body = (
117+
response.peek()
118+
if getattr(response, 'peek', None)
119+
else response.data
120+
if response.tell() > 0
121+
else None
122+
)
117123
if isinstance(response_body, bytes):
118124
try:
119125
response_body = response_body.decode('utf-8')

epsagon/modules/urllib.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"""
44

55
from __future__ import absolute_import
6-
import wrapt
6+
from epsagon.utils import patch_once
77
from epsagon.modules.general_wrapper import wrapper
88
from ..events.urllib import UrllibEventFactory
99

@@ -27,7 +27,7 @@ def patch():
2727
"""
2828

2929
try:
30-
wrapt.wrap_function_wrapper(
30+
patch_once(
3131
'urllib.request',
3232
'OpenerDirector._open',
3333
_wrapper

epsagon/modules/urllib3.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
from __future__ import absolute_import
66
import uuid
7-
import wrapt
7+
from epsagon.utils import patch_once
88
from epsagon.modules.general_wrapper import wrapper
99
from ..events.urllib3 import Urllib3EventFactory
1010
from ..http_filters import is_blacklisted_url
@@ -82,7 +82,7 @@ def patch():
8282
"""
8383

8484
try:
85-
wrapt.wrap_function_wrapper(
85+
patch_once(
8686
'urllib3',
8787
'HTTPConnectionPool.urlopen',
8888
_wrapper

tests/modules/test_urllib3.py

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
import pytest
2+
import urllib3
3+
import epsagon.wrappers.python_function
4+
from epsagon.trace import trace_factory
5+
import epsagon.runners.python_function
6+
import epsagon.constants
7+
8+
9+
TEST_DOMAIN = 'jsonplaceholder.typicode.com'
10+
TEST_PATH = '/todos/1'
11+
12+
def setup_function(func):
13+
trace_factory.use_single_trace = True
14+
epsagon.constants.COLD_START = True
15+
16+
def test_no_data_capture_with_urlopen(trace_transport):
17+
def use_urlopen():
18+
headers = {
19+
'Content-Type': 'application/json'
20+
}
21+
22+
http = urllib3.PoolManager()
23+
conn = http.connection_from_url(TEST_DOMAIN)
24+
urllib_response = conn.urlopen(
25+
method='GET',
26+
url=TEST_PATH,
27+
body='',
28+
headers=headers,
29+
preload_content=False,
30+
decode_content=False,
31+
)
32+
return urllib_response
33+
34+
response = use_urlopen()
35+
data = response.read()
36+
37+
@epsagon.wrappers.python_function.python_wrapper
38+
def wrapped_function():
39+
return use_urlopen()
40+
41+
wrapped_response = wrapped_function()
42+
wrapped_data = wrapped_response.read()
43+
# We expect in this case that the data will be available in the buffer
44+
assert(len(wrapped_data) > 0)
45+
assert(len(wrapped_response.data) == 0)
46+
urllib3_event = trace_transport.last_trace.events[-1]
47+
# Payload should not be collected in the case of raw-stream usage.
48+
assert(urllib3_event.resource['metadata']['response_body'] is None)
49+
# Compare un-instrumented vs instrumented data
50+
assert (len(data) > 0)
51+
assert (len(response.data) == 0)
52+
assert(wrapped_data == data)
53+
assert(wrapped_response.data == response.data)
54+
55+
@pytest.mark.parametrize("preload_content", [True, False])
56+
def test_data_capture_with_pool_manager(preload_content, trace_transport):
57+
def use_poolmanager():
58+
headers = {
59+
'Content-Type': 'application/json'
60+
}
61+
62+
http = urllib3.PoolManager()
63+
return http.request(
64+
'GET', TEST_DOMAIN + TEST_PATH,
65+
headers=headers, body='',
66+
preload_content=preload_content
67+
)
68+
69+
response = use_poolmanager()
70+
data = response.read()
71+
72+
@epsagon.wrappers.python_function.python_wrapper
73+
def wrapped_function():
74+
return use_poolmanager()
75+
76+
wrapped_response = wrapped_function()
77+
wrapped_data = wrapped_response.read()
78+
if not preload_content:
79+
# In this case data will not be available in the buffer
80+
assert (len(wrapped_data) > 0)
81+
# But, data will not be stored in the `data` object
82+
assert (len(wrapped_response.data) == 0)
83+
# Compare un-instrumented vs instrumented data
84+
assert (len(data) > 0)
85+
assert (len(response.data) == 0)
86+
else:
87+
# In this case data will not be available in the buffer
88+
assert (len(wrapped_data) == 0)
89+
# But, data will be stored in the `data` object
90+
assert (len(wrapped_response.data) > 0)
91+
# Compare un-instrumented vs instrumented data
92+
assert (len(data) == 0)
93+
assert (len(response.data) > 0)
94+
assert(wrapped_data == data)
95+
assert(wrapped_response.data == response.data)

tests/test_utils.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import pytest
12
import epsagon.trace
23
import epsagon.utils
34
import epsagon.http_filters
@@ -7,8 +8,13 @@
78
def setup_function(func):
89
trace_factory.get_or_create_trace()
910

11+
@pytest.yield_fixture
12+
def blacklist_urls_state_keeper():
13+
original = epsagon.http_filters.BLACKLIST_URLS # Storing old state
14+
yield
15+
epsagon.http_filters.BLACKLIST_URLS = original # Restoring old state
1016

11-
def test_blacklist_url():
17+
def test_blacklist_url(blacklist_urls_state_keeper):
1218
"""
1319
Test is_blacklisted_url functionality.
1420
:return: None
@@ -36,7 +42,6 @@ def test_original_blacklist_url():
3642
Validate original needed URLs are in.
3743
:return: None
3844
"""
39-
4045
assert epsagon.http_filters.is_blacklisted_url('http://tc.us-east-1.epsagon.com')
4146
assert epsagon.http_filters.is_blacklisted_url('https://client.tc.epsagon.com')
4247

0 commit comments

Comments
 (0)