8000 feat: Django sql queries (#94) · etherscan-io/sentry-python@71294f5 · GitHub
[go: up one dir, main page]
Skip to content

Commit 71294f5

Browse files
untitakeruntitaker
authored
feat: Django sql queries (getsentry#94)
* feat: Django sql queries * fix: Handle too large sql params better * fix: Fix test
1 parent eb2c1be commit 71294f5

File tree

6 files changed

+245
-15
lines changed

6 files changed

+245
-15
lines changed

‎sentry_sdk/_compat.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
string_types = (str, text_type)
1313
number_types = (int, long, float) # noqa
14+
int_types = (int, long) # noqa
1415

1516
def implements_str(cls):
1617
cls.__unicode__ = cls.__str__
@@ -32,6 +33,7 @@ def implements_iterator(cls):
3233
text_type = str
3334
string_types = (text_type,)
3435
number_types = (int, float)
36+
int_types = (int,) # noqa
3537

3638
def _identity(x):
3739
return x

‎sentry_sdk/integrations/django.py

Lines changed: 91 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# -*- coding: utf-8 -*-
12
from __future__ import absolute_import
23

34
import sys
@@ -11,10 +12,16 @@
1112
except ImportError:
1213
from django.core.urlresolvers import resolve
1314

14-
from sentry_sdk import Hub, configure_scope
15+
from sentry_sdk import Hub, configure_scope, add_breadcrumb
1516
from sentry_sdk.hub import _should_send_default_pii
16-
from sentry_sdk.utils import capture_internal_exceptions, event_from_exception
17+
from sentry_sdk.utils import (
18+
capture_internal_exceptions,
19+
event_from_exception,
20+
safe_repr,
21+
format_and_strip,
22+
)
1723
from sentry_sdk.integrations import Integration
24+
from sentry_sdk.integrations.logging import ignore_logger
1825
from sentry_sdk.integrations._wsgi import RequestExtractor, run_wsgi_app
1926

2027

@@ -34,6 +41,7 @@ class DjangoIntegration(Integration):
3441
identifier = "django"
3542

3643
def install(self):
44+
install_sql_hook()
3745
# Patch in our custom middleware.
3846

3947
from django.core.handlers.wsgi import WSGIHandler
@@ -141,3 +149,84 @@ def _set_user_info(request, event):
141149
user_info["username"] = user.get_username()
142150
except Exception:
143151
pass
152+
153+
154+
class _FormatConverter(object):
155+
def __init__(self, param_mapping):
156+
self.param_mapping = param_mapping
157+
self.params = []
158+
159+
def __getitem__(self, val):
160+
self.params.append(self.param_mapping.get(val))
161+
return "%s"
162+
163+
164+
def format_sql(sql, params):
165+
rv = []
166+
167+
if isinstance(params, dict):
168+
# convert sql with named parameters to sql with unnamed parameters
169+
conv = _FormatConverter(params)
170+
if params:
171+
sql = sql % conv
172+
params = conv.params
173+
else:
174+
params = ()
175+
176+
for param in params or ():
177+
if param is None:
178+
rv.append("NULL")
179+
param = safe_repr(param)
180+
rv.append(param)
181+
182+
return sql, rv
183+
184+
185+
def record_sql(sql, params):
186+
real_sql, real_params = format_sql(sql, params)
187+
188+
if real_params:
189+
try:
190+
real_sql = format_and_strip(real_sql, real_params)
191+
except Exception:
192+
pass
193+
194+
# maybe category to 'django.%s.%s' % (vendor, alias or
195+
# 'default') ?
196+
197+
add_breadcrumb(message=real_sql, category="query")
198+
199+
200+
def install_sql_hook():
201+
"""If installed this causes Django's queries to be captured."""
202+
try:
203+
from django.db.backends.utils import CursorWrapper
204+
except ImportError:
205+
from django.db.backends.util import CursorWrapper
206+
207+
try:
208+
real_execute = CursorWrapper.execute
209+
real_executemany = CursorWrapper.executemany
210+
except AttributeError:
211+
# This won't work on Django versions < 1.6
212+
return
213+
214+
def record_many_sql(sql, param_list):
215+
for params in param_list:
216+
record_sql(sql, params)
217+
218+
def execute(self, sql, params=None):
219+
try:
220+
return real_execute(self, sql, params)
221+
finally:
222+
record_sql(sql, params)
223+
224+
def executemany(self, sql, param_list):
225+
try:
226+
return real_executemany(self, sql, param_list)
227+
finally:
228+
record_many_sql(sql, param_list)
229+
230+
CursorWrapper.execute = execute
231+
CursorWrapper.executemany = executemany
232+
ignore_logger("django.db.backends")

‎sentry_sdk/utils.py

Lines changed: 66 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
implements_str,
1414
string_types,
1515
number_types,
16+
int_types,
1617
)
1718

1819

@@ -499,10 +500,10 @@ def inner(obj):
499500
rv = []
500501
meta = {}
501502
for i, v in enumerate(obj):
502-
new_v, meta[i] = inner(v)
503+
new_v, meta[str(i)] = inner(v)
503504
rv.append(new_v)
504-
if meta[i] is None:
505-
del meta[i]
505+
if meta[str(i)] is None:
506+
del meta[str(i)]
506507
return rv, (meta or None)
507508
if isinstance(obj, AnnotatedValue):
508509
return obj.value, {"": obj.metadata}
@@ -557,22 +558,77 @@ def strip_databag(obj, remaining_depth=20):
557558
return obj
558559

559560

560-
def strip_string(value, assume_length=None, max_length=512):
561+
def strip_string(value, max_length=512):
561562
# TODO: read max_length from config
562563
if not value:
563564
return value
564-
if assume_length is None:
565-
assume_length = len(value)
566-
567-
if assume_length > max_length:
565+
length = len(value)
566+
if length > max_length:
568567
return AnnotatedValue(
569568
value=value[: max_length - 3] + u"...",
570569
metadata={
571-
"len": assume_length,
570+
"len": leng F987 th,
572571
"rem": [["!limit", "x", max_length - 3, max_length]],
573572
},
574573
)
575-
return value[:max_length]
574+
return value
575+
576+
577+
def format_and_strip(template, params, strip_string=strip_string):
578+
"""Format a string containing %s for placeholders and call `strip_string`
579+
on each parameter. The string template itself does not have a maximum
580+
length.
581+
582+
TODO: handle other placeholders, not just %s
583+
"""
584+
chunks = template.split(u"%s")
585+
if not chunks:
586+
raise ValueError("No formatting placeholders found")
587+
588+
params = list(reversed(params))
589+
rv_remarks = []
590+
rv_original_length = 0
591+
rv_length = 0
592+
rv = []
593+
594+
def realign_remark(remark):
595+
return [
596+
(rv_length + x if isinstance(x, int_types) and i < 4 else x)
597+
for i, x in enumerate(remark)
598+
]
599+
600+
for chunk in chunks[:-1]:
601+
rv.append(chunk)
602+
rv_length += len(chunk)
603+
rv_original_length += len(chunk)
604+
if not params:
605+
raise ValueError("Not enough params.")
606+
param = params.pop()
607+
608+
stripped_param = strip_string(param)
609+
if isinstance(stripped_param, AnnotatedValue):
610+
rv_remarks.extend(
611+
realign_remark(remark) for remark in stripped_param.metadata["rem"]
612+
)
613+
stripped_param = stripped_param.value
614+
615+
rv_original_length += len(param)
616+
rv_length += len(stripped_param)
617+
rv.append(stripped_param)
618+
619+
rv.append(chunks[-1])
620+
rv_length += len(chunks[-1])
621+
rv_original_length += len(chunks[-1])
622+
623+
rv = u"".join(rv)
624+
assert len(rv) == rv_length
625+
626+
if not rv_remarks:
627+
return rv
628+
629+
return AnnotatedValue(
630+
value=rv, metadata={"len": rv_original_length, "rem": rv_remarks}
631+
)
576632

577633

578634
try:

‎tests/integrations/django/test_basic.py

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
except ImportError:
1313
from django.core.urlresolvers import reverse
1414

15-
from sentry_sdk import last_event_id
15+
from sentry_sdk import last_event_id, capture_message
1616
from sentry_sdk.integrations.django import DjangoIntegration
1717

1818
from tests.integrations.django.myapp.wsgi import application
@@ -103,3 +103,46 @@ def test_management_command_raises():
103103
# commands.
104104
with pytest.raises(ZeroDivisionError):
105105
execute_from_command_line(["manage.py", "mycrash"])
106+
107+
108+
@pytest.mark.django_db
109+
def test_sql_queries(capture_events):
110+
from django.db import connection
111+
112+
sql = connection.cursor()
113+
114+
events = capture_events()
115+
with pytest.raises(Exception):
116+
# table doesn't even exist
117+
sql.execute("""SELECT count(*) FROM people_person WHERE foo = %s""", [123])
118+
119+
capture_message("HI")
120+
121+
event, = events
122+
123+
crumb, = event["breadcrumbs"]
124+
125+
assert crumb["message"] == """SELECT count(*) FROM people_person WHERE foo = 123"""
126+
127+
128+
@pytest.mark.django_db
129+
def test_sql_queries_large_params(capture_events):
130+
from django.db import connection
131+
132+
sql = connection.cursor()
133+
134+
events = capture_events()
135+
with pytest.raises(Exception):
136+
# table doesn't even exist
137+
sql.execute(
138+
"""SELECT count(*) FROM people_person WHERE foo = %s""", ["x" * 1000]
139+
)
140+
141+
capture_message("HI")
142+
143+
event, = events
144+
145+
crumb, = event["breadcrumbs"]
146+
assert crumb["message"] == (
147+
"SELECT count(*) FROM people_person WHERE foo = '%s..." % ("x" * 508,)
148+
)

‎tests/test_event.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ def test_flatten_metadata():
66
assert flatten_metadata({"foo": ["bar"]}) == {"foo": [u"bar"]}
77
assert flatten_metadata({"foo": [AnnotatedValue("bar", u"meta")]}) == {
88
"foo": [u"bar"],
9-
"": {"foo": {0: {"": u"meta"}}},
9+
"": {"foo": {"0": {"": u"meta"}}},
1010
}
1111

1212

‎tests/test_utils.py

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,17 @@
22
import sys
33
import os
44

5+
import pytest
6+
57
from hypothesis import given
68
import hypothesis.strategies as st
79

8-
from sentry_sdk.utils import safe_repr, exceptions_from_error_tuple
10+
from sentry_sdk.utils import (
11+
safe_repr,
12+
exceptions_from_error_tuple,
13+
format_and_strip,
14+
strip_string,
15+
)
916
from sentry_sdk._compat import text_type
1017

1118
any_string = st.one_of(st.binary(), st.text())
@@ -56,3 +63,36 @@ def test_non_string_variables():
5663
assert exception["type"] == "ZeroDivisionError"
5764
frame, = exception["stacktrace"]["frames"]
5865
assert frame["vars"]["42"] == "True"
66+
67+
68+
def test_format_and_strip():
69+
max_length = None
70+
71+
def x(template, params):
72+
return format_and_strip(
73+
template,
74+
params,
75+
strip_string=lambda x: strip_string(x, max_length=max_length),
76+
)
77+
78+
max_length = 3
79+
80+
assert x("", []) == ""
81+
assert x("f", []) == "f"
82+
pytest.raises(ValueError, lambda: x("%s", []))
83+
84+
# Don't raise errors on leftover params, some django extensions send too
85+
# many SQL parameters.
86+
assert x("", [123]) == ""
87+
assert x("foo%s", ["bar"]) == "foobar"
88+
89+
rv = x("foo%s", ["baer"])
90+
assert rv.value == "foo..."
91+
assert rv.metadata == {"len": 7, "rem": [["!limit", "x", 3, 6]]}
92+
93+
rv = x("foo%sbar%s", ["baer", "boor"])
94+
assert rv.value == "foo...bar..."
95+
assert rv.metadata == {
96+
"len": 14,
97+
"rem": [["!limit", "x", 3, 6], ["!limit", "x", 9, 12]],
98+
}

0 commit comments

Comments
 (0)
0