8000 Improve object stats (#92845) · python/cpython@fa2b8b7 · GitHub
[go: up one dir, main page]
Skip to content

Commit fa2b8b7

Browse files
markshannonmarkshannon
authored
Improve object stats (#92845)
* Add incref/decref stats * Show ratios for allocation in summary
1 parent f6fd8aa commit fa2b8b7

File tree

10 files changed

+106
-55
lines changed

10 files changed

+106
-55
lines changed

‎Include/internal/pycore_code.h

Lines changed: 0 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -265,53 +265,6 @@ extern int _PyStaticCode_InternStrings(PyCodeObject *co);
265265

266266
#ifdef Py_STATS
267267

268-
#define SPECIALIZATION_FAILURE_KINDS 30
269-
270-
typedef struct _specialization_stats {
271-
uint64_t success;
272-
uint64_t failure;
273-
uint64_t hit;
274-
uint64_t deferred;
275-
uint64_t miss;
276-
uint64_t deopt;
277-
uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
278-
} SpecializationStats;
279-
280-
typedef struct _opcode_stats {
281-
SpecializationStats specialization;
282-
uint64_t execution_count;
283-
uint64_t pair_count[256];
284-
} OpcodeStats;
285-
286-
typedef struct _call_stats {
287-
uint64_t inlined_py_calls;
288-
uint64_t pyeval_calls;
289-
uint64_t frames_pushed;
290-
uint64_t frame_objects_created;
291-
} CallStats;
292-
293-
typedef struct _object_stats {
294-
uint64_t allocations;
295-
uint64_t allocations512;
296-
uint64_t allocations4k;
297-
uint64_t allocations_big;
298-
uint64_t frees;
299-
uint64_t to_freelist;
300-
uint64_t from_freelist;
301-
uint64_t new_values;
302-
uint64_t dict_materialized_on_request;
303-
uint64_t dict_materialized_new_key;
304-
uint64_t dict_materialized_too_big;
305-
uint64_t dict_materialized_str_subclass;
306-
} ObjectStats;
307-
308-
typedef struct _stats {
309-
OpcodeStats opcode_stats[256];
310-
CallStats call_stats;
311-
ObjectStats object_stats;
312-
} PyStats;
313-
314-
extern PyStats _py_stats;
315268

316269
#define STAT_INC(opname, name) _py_stats.opcode_stats[opname].specialization.name++
317270
#define STAT_DEC(opname, name) _py_stats.opcode_stats[opname].specialization.name--
@@ -321,8 +274,6 @@ extern PyStats _py_stats;
321274
#define OBJECT_STAT_INC_COND(name, cond) \
322275
do { if (cond) _py_stats.object_stats.name++; } while (0)
323276

324-
extern void _Py_PrintSpecializationStats(int to_file);
325-
326277
// Used by the _opcode extension which is built as a shared library
327278
PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
328279

‎Include/internal/pycore_object.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
3434
static inline void
3535
_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
3636
{
37+
_Py_DECREF_STAT_INC();
3738
#ifdef Py_REF_DEBUG
3839
_Py_RefTotal--;
3940
#endif
@@ -51,6 +52,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
5152
static inline void
5253
_Py_DECREF_NO_DEALLOC(PyObject *op)
5354
{
55+
_Py_DECREF_STAT_INC();
5456
#ifdef Py_REF_DEBUG
5557
_Py_RefTotal--;
5658
#endif

‎Include/object.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ A standard interface exists for objects that contain an array of items
5151
whose size is determined when the object is allocated.
5252
*/
5353

54+
#include "pystats.h"
55+
5456
/* Py_DEBUG implies Py_REF_DEBUG. */
5557
#if defined(Py_DEBUG) && !defined(Py_REF_DEBUG)
5658
# define Py_REF_DEBUG
@@ -490,6 +492,7 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *);
490492

491493
static inline void Py_INCREF(PyObject *op)
492494
{
495+
_Py_INCREF_STAT_INC();
493496
#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
494497
// Stable ABI for Python 3.10 built in debug mode.
495498
_Py_IncRef(op);
@@ -506,7 +509,6 @@ static inline void Py_INCREF(PyObject *op)
506509
# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op))
507510
#endif
508511

509-
510512
#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
511513
// Stable ABI for limited C API version 3.10 of Python debug build
512514
static inline void Py_DECREF(PyObject *op) {
@@ -517,6 +519,7 @@ static inline void Py_DECREF(PyObject *op) {
517519
#elif defined(Py_REF_DEBUG)
518520
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
519521
{
522+
_Py_DECREF_STAT_INC();
520523
_Py_RefTotal--;
521524
if (--op->ob_refcnt != 0) {
522525
if (op->ob_refcnt < 0) {
@@ -532,6 +535,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
532535
#else
533536
static inline void Py_DECREF(PyObject *op)
534537
{
538+
_Py_DECREF_STAT_INC();
535539
// Non-limited C API and limited C API for Python 3.9 and older access
536540
// directly PyObject.ob_refcnt.
537541
if (--op->ob_refcnt == 0) {

‎Include/pystats.h

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
2+
3+
#ifndef Py_PYSTATS_H
4+
#define Py_PYSTATS_H
5+
#ifdef __cplusplus
6+
extern "C" {
7+
#endif
8+
9+
#ifdef Py_STATS
10+
11+
#define SPECIALIZATION_FAILURE_KINDS 32
12+
13+
typedef struct _specialization_stats {
14+
uint64_t success;
15+
uint64_t failure;
16+
uint64_t hit;
17+
uint64_t deferred;
18+
uint64_t miss;
19+
uint64_t deopt;
20+
uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
21+
} SpecializationStats;
22+
23+
typedef struct _opcode_stats {
24+
SpecializationStats specialization;
25+
uint64_t execution_count;
26+
uint64_t pair_count[256];
27+
} OpcodeStats;
28+
29+
typedef struct _call_stats {
30+
uint64_t inlined_py_calls;
31+
uint64_t pyeval_calls;
32+
uint64_t frames_pushed;
33+
uint64_t frame_objects_created;
34+
} CallStats;
35+
36+
typedef struct _object_stats {
37+
uint64_t increfs;
38+
uint64_t decrefs;
39+
uint64_t allocations;
40+
uint64_t allocations512;
41+
uint64_t allocations4k;
42+
uint64_t allocations_big;
43+
uint64_t frees;
44+
uint64_t to_freelist;
45+
uint64_t from_freelist;
46+
uint64_t new_values;
47+
uint64_t dict_materialized_on_request;
48+
uint64_t dict_materialized_new_key;
49+
uint64_t dict_materialized_too_big;
50+
uint64_t dict_materialized_str_subclass;
51+
} ObjectStats;
52+
53+
typedef struct _stats {
54+
OpcodeStats opcode_stats[256];
55+
CallStats call_stats;
56+
ObjectStats object_stats;
57+
} PyStats;
58+
59+
PyAPI_DATA(PyStats) _py_stats;
60+
61+
extern void _Py_PrintSpecializationStats(int to_file);
62+
63+
64+
#define _Py_INCREF_STAT_INC() _py_stats.object_stats.increfs++
65+
#define _Py_DECREF_STAT_INC() _py_stats.object_stats.decrefs++
66+
67+
#else
68+
69+
#define _Py_INCREF_STAT_INC() ((void)0)
70+
#define _Py_DECREF_STAT_INC() ((void)0)
71+
72+
#endif // !Py_STATS
73+
74+
#ifdef __cplusplus
75+
}
76+
#endif
77+
#endif /* !Py_PYSTATs_H */

‎Makefile.pre.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1509,6 +1509,7 @@ PYTHON_HEADERS= \
15091509
$(srcdir)/Include/pymem.h \
15101510
$(srcdir)/Include/pyport.h \
15111511
$(srcdir)/Include/pystate.h \
1512+
$(srcdir)/Include/pystats.h \
15121513
$(srcdir)/Include/pystrcmp.h \
15131514
$(srcdir)/Include/pystrtod.h \
15141515
$(srcdir)/Include/pythonrun.h \

‎PCbuild/pythoncore.vcxproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,7 @@
280280
<ClInclude Include="..\Include\pymem.h" />
281281
<ClInclude Include="..\Include\pyport.h" />
282282
<ClInclude Include="..\Include\pystate.h" />
283+
<ClInclude Include="..\Include\pystats.h" />
283284
<ClInclude Include="..\Include\pystrcmp.h" />
284285
<ClInclude Include="..\Include\pystrtod.h" />
285286
<ClInclude Include="..\Include\pythonrun.h" />

‎PCbuild/pythoncore.vcxproj.filters

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,9 @@
174174
<ClInclude Include="..\Include\pystate.h">
175175
<Filter>Include</Filter>
176176
</ClInclude>
177+
<ClInclude Include="..\Include\pystats.h">
178+
<Filter>Include</Filter>
179+
</ClInclude>
177180
<ClInclude Include="..\Include\pystrcmp.h">
178181
<Filter>Include</Filter>
179182
</ClInclude>

‎Python/ceval.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
#undef Py_DECREF
5656
#define Py_DECREF(arg) \
5757
do { \
58+
_Py_DECREF_STAT_INC(); \
5859
PyObject *op = _PyObject_CAST(arg); \
5960
if (--op->ob_refcnt == 0) { \
6061
destructor dealloc = Py_TYPE(op)->tp_dealloc; \
@@ -78,6 +79,7 @@
7879
#undef _Py_DECREF_SPECIALIZED
7980
#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
8081
do { \
82+
_Py_DECREF_STAT_INC(); \
8183
PyObject *op = _PyObject_CAST(arg); \
8284
if (--op->ob_refcnt == 0) { \
8385
destructor d = (destructor)(dealloc); \

‎Python/specialize.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,8 @@ print_object_stats(FILE *out, ObjectStats *stats)
191191
fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
192192
fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
193193
fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
194+
fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
195+
fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs);
194196
fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
195197
fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
196198
fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);

‎Tools/scripts/summarize_stats.py

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from datetime import date
99
import itertools
1010
import argparse
11+
import sys
1112

1213
if os.name == "nt":
1314
DEFAULT_DIR = "c:\\temp\\py_stats\\"
@@ -88,7 +89,11 @@ def gather_stats():
8889
for filename in os.listdir(DEFAULT_DIR):
8990
with open(os.path.join(DEFAULT_DIR, filename)) as fd:
9091
for line in fd:
91-
key, value = line.split(":")
92+
try:
93+
key, value = line.split(":")
94+
except ValueError:
95+
print (f"Unparsable line: '{line.strip()}' in {filename}", file=sys.stderr)
96+
continue
9297
key = key.strip()
9398
value = int(value)
9499
stats[key] += value
@@ -265,17 +270,20 @@ def emit_call_stats(stats):
265270

266271
def emit_object_stats(stats):
267272
with Section("Object stats", summary="allocations, frees and dict materializatons"):
268-
total = stats.get("Object new values")
273+
total_materializations = stats.get("Object new values")
274+
total_allocations = stats.get("Object allocations")
269275
rows = []
270276
for key, value in stats.items():
271277
if key.startswith("Object"):
272278
if "materialize" in key:
273-
materialize = f"{100*value/total:0.1f}%"
279+
ratio = f"{100*value/total_materializations:0.1f}%"
280+
elif "allocations" in key:
281+
ratio = f"{100*value/total_allocations:0.1f}%"
274282
else:
275-
materialize = ""
283+
ratio = ""
276284
label = key[6:].strip()
277285
label = label[0].upper() + label[1:]
278-
rows.append((label, value, materialize))
286+
rows.append((label, value, ratio))
279287
emit_table(("", "Count:", "Ratio:"), rows)
280288

281289
def get_total(opcode_stats):

0 commit comments

Comments
 (0)
0