From e234c1fe277d321a22b00836e752d5509234d190 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 12 Feb 2024 14:21:17 -0500 Subject: [PATCH 1/5] gh-115362: Add documentation to pystats output --- Tools/scripts/summarize_stats.py | 102 ++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 15 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 7891b9cf923d33..99d0ab2de9bd3c 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -21,6 +21,7 @@ from pathlib import Path import re import sys +import textwrap from typing import Any, Callable, TextIO, TypeAlias @@ -389,17 +390,58 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: low_confidence = self._data["Optimization low confidence"] return { - "Optimization attempts": (attempts, None), - "Traces created": (created, attempts), - "Trace stack overflow": (trace_stack_overflow, attempts), - "Trace stack underflow": (trace_stack_underflow, attempts), - "Trace too long": (trace_too_long, attempts), - "Trace too short": (trace_too_short, attempts), - "Inner loop found": (inner_loop, attempts), - "Recursive call": (recursive_call, attempts), - "Low confidence": (low_confidence, attempts), - "Traces executed": (executed, None), - "Uops executed": (uops, executed), + Doc( + "Optimization attempts", + """ + The number of times a potential trace is identified. Specifically, + this occurs in the JUMP BACKWARD instruction when the counter reaches + a threshold. + """, + ): ( + attempts, + None, + ), + Doc( + "Traces created", "The number of traces that were successfully created." + ): (created, attempts), + Doc( + "Trace stack overflow", + "A trace is truncated because it would require more than 5 stack frames.", + ): (trace_stack_overflow, attempts), + Doc( + "Trace stack underflow", + "A potential trace is abandoned because it pops more frames than it pushes.", + ): (trace_stack_underflow, attempts), + Doc( + "Trace too long", + "A trace is truncated because it is longer than the instruction buffer.", + ): (trace_too_long, attempts), + Doc( + "Trace too short", + "A potential trace is abandoced because it it too short.", + ): (trace_too_short, attempts), + Doc( + "Inner loop found", "A trace is truncated because it has an inner loop" + ): (inner_loop, attempts), + Doc( + "Recursive call", + "A trace is truncated because it has a recursive call.", + ): (recursive_call, attempts), + Doc( + "Low confidence", + """ + A trace is abandoned because the likelihood of the jump to top being + taken is too low. + """, + ): (low_confidence, attempts), + Doc("Traces executed", "The number of traces that were executed"): ( + executed, + None, + ), + Doc("Uops executed", "The total number of uops that were executed"): ( + uops, + executed, + ), } def get_histogram(self, prefix: str) -> list[tuple[int, int]]: @@ -415,12 +457,21 @@ def get_histogram(self, prefix: str) -> list[tuple[int, int]]: def get_rare_events(self) -> list[tuple[str, int]]: prefix = "Rare event " return [ - (key[len(prefix) + 1:-1].replace("_", " "), val) + (key[len(prefix) + 1 : -1].replace("_", " "), val) for key, val in self._data.items() if key.startswith(prefix) ] +class Doc: + def __init__(self, text: str, doc: str): + self.text = text + self.doc = textwrap.dedent(doc).strip() + + def markdown(self) -> str: + return f'{self.text} [ⓘ](## "{self.doc}")' + + class Count(int): def markdown(self) -> str: return format(self, ",d") @@ -568,13 +619,16 @@ def __init__( title: str = "", summary: str = "", part_iter=None, + *, comparative: bool = True, + doc: str = "", ): self.title = title if not summary: self.summary = title.lower() else: self.summary = summary + self.doc = textwrap.dedent(doc) if part_iter is None: part_iter = [] if isinstance(part_iter, list): @@ -628,6 +682,10 @@ def execution_count_section() -> Section: join_mode=JoinMode.CHANGE_ONE_COLUMN, ) ], + doc=""" + The "miss ratio" column shows the percentage of times the instruction + executed that it deoptimized. + """, ) @@ -655,7 +713,7 @@ def calc_pair_count_table(stats: Stats) -> Rows: return Section( "Pair counts", - "Pair counts for top 100 pairs", + "Pair counts for top 100 Tier 1 instructions", [ Table( ("Pair", "Count:", "Self:", "Cumulative:"), @@ -705,7 +763,7 @@ def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = Non return Section( "Predecessor/Successor Pairs", - "Top 5 predecessors and successors of each opcode", + "Top 5 predecessors and successors of each Tier 1 opcode", iter_pre_succ_pairs_tables, comparative=False, ) @@ -1073,8 +1131,19 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) def rare_event_section() -> Section: + RARE_DOCS = { + "set class": "Setting an object's class, `obj.__class__ = ...`", + "set bases": "Setting the bases of a class, `cls.__bases__ = ...`", + "set eval frame func": ( + "Setting the PEP 523 frame eval function " + "`_PyInterpreterState_SetFrameEvalFunc()`" + ), + "builtin dict": "Modifying the builtins, `__builtins__.__dict__[var] = ...`", + "func modification": "Modifying a function, e.g. `func.__defaults__ = ...`, etc.", + } + def calc_rare_event_table(stats: Stats) -> Table: - return [(x, Count(y)) for x, y in stats.get_rare_events()] + return [(Doc(x, RARE_DOCS[x]), Count(y)) for x, y in stats.get_rare_events()] return Section( "Rare events", @@ -1134,6 +1203,9 @@ def to_markdown(x): print("
", file=out) print("", obj.summary, "", file=out) print(file=out) + if obj.doc: + print(obj.doc, file=out) + if head_stats is not None and obj.comparative is False: print("Not included in comparative output.\n") else: From 684abbfff8727ea217055e18cd7806264a4666f1 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 12 Feb 2024 14:26:31 -0500 Subject: [PATCH 2/5] Fix tooltips --- Tools/scripts/summarize_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 99d0ab2de9bd3c..a4193b568f97a8 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -466,7 +466,7 @@ def get_rare_events(self) -> list[tuple[str, int]]: class Doc: def __init__(self, text: str, doc: str): self.text = text - self.doc = textwrap.dedent(doc).strip() + self.doc = textwrap.dedent(doc).strip().replace("\n", " ") def markdown(self) -> str: return f'{self.text} [ⓘ](## "{self.doc}")' From c229cae23089e63068086be9bd62c4cbda9c53d0 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 15 Feb 2024 12:35:49 -0500 Subject: [PATCH 3/5] Complete set of docs. Use
instead of tooltips. --- Tools/scripts/summarize_stats.py | 262 ++++++++++++++++++++----------- 1 file changed, 168 insertions(+), 94 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index a4193b568f97a8..68bb2b46bb0a03 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -116,6 +116,64 @@ def save_raw_data(data: RawData, json_output: TextIO): json.dump(data, json_output) +class Doc: + def __init__(self, text: str, doc: str): + self.text = text + self.doc = doc + + def markdown(self) -> str: + return textwrap.dedent( + f""" + {self.text} +
+ ⓘ + + {self.doc} +
+ """ + ) + + +class Count(int): + def markdown(self) -> str: + return format(self, ",d") + + +class Ratio: + def __init__(self, num: int, den: int | None, percentage: bool = True): + self.num = num + self.den = den + self.percentage = percentage + + def __float__(self): + if self.den == 0: + return 0.0 + elif self.den is None: + return self.num + else: + return self.num / self.den + + def markdown(self) -> str: + if self.den is None: + return "" + elif self.den == 0: + if self.num != 0: + return f"{self.num:,} / 0 !!" + return "" + elif self.percentage: + return f"{self.num / self.den:,.01%}" + else: + return f"{self.num / self.den:,.02f}" + + +class DiffRatio(Ratio): + def __init__(self, base: int | str, head: int | str): + if isinstance(base, str) or isinstance(head, str): + super().__init__(0, 0) + else: + super().__init__(head - base, base) + + class OpcodeStats: """ Manages the data related to specific set of opcodes, e.g. tier1 (with prefix @@ -392,11 +450,9 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: return { Doc( "Optimization attempts", - """ - The number of times a potential trace is identified. Specifically, - this occurs in the JUMP BACKWARD instruction when the counter reaches - a threshold. - """, + "The number of times a potential trace is identified. Specifically, this " + "occurs in the JUMP BACKWARD instruction when the counter reaches a " + "threshold.", ): ( attempts, None, @@ -429,10 +485,8 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: ): (recursive_call, attempts), Doc( "Low confidence", - """ - A trace is abandoned because the likelihood of the jump to top being - taken is too low. - """, + "A trace is abandoned because the likelihood of the jump to top being taken " + "is too low.", ): (low_confidence, attempts), Doc("Traces executed", "The number of traces that were executed"): ( executed, @@ -463,55 +517,6 @@ def get_rare_events(self) -> list[tuple[str, int]]: ] -class Doc: - def __init__(self, text: str, doc: str): - self.text = text - self.doc = textwrap.dedent(doc).strip().replace("\n", " ") - - def markdown(self) -> str: - return f'{self.text} [ⓘ](## "{self.doc}")' - - -class Count(int): - def markdown(self) -> str: - return format(self, ",d") - - -class Ratio: - def __init__(self, num: int, den: int | None, percentage: bool = True): - self.num = num - self.den = den - self.percentage = percentage - - def __float__(self): - if self.den == 0: - return 0.0 - elif self.den is None: - return self.num - else: - return self.num / self.den - - def markdown(self) -> str: - if self.den is None: - return "" - elif self.den == 0: - if self.num != 0: - return f"{self.num:,} / 0 !!" - return "" - elif self.percentage: - return f"{self.num / self.den:,.01%}" - else: - return f"{self.num / self.den:,.02f}" - - -class DiffRatio(Ratio): - def __init__(self, base: int | str, head: int | str): - if isinstance(base, str) or isinstance(head, str): - super().__init__(0, 0) - else: - super().__init__(head - base, base) - - class JoinMode(enum.Enum): # Join using the first column as a key SIMPLE = 0 @@ -674,7 +679,7 @@ def calc(stats: Stats) -> Rows: def execution_count_section() -> Section: return Section( "Execution counts", - "execution counts for all instructions", + "Execution counts for Tier 1 instructions.", [ Table( ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), @@ -684,7 +689,8 @@ def execution_count_section() -> Section: ], doc=""" The "miss ratio" column shows the percentage of times the instruction - executed that it deoptimized. + executed that it deoptimized. When this happens, the base unspecialized + instruction is not counted. """, ) @@ -721,6 +727,10 @@ def calc_pair_count_table(stats: Stats) -> Rows: ) ], comparative=False, + doc=""" + Pairs of specialized operations that deoptimize and are then followed by + the corresponding unspecialized instruction are not counted as pairs. + """, ) @@ -763,22 +773,33 @@ def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = Non return Section( "Predecessor/Successor Pairs", - "Top 5 predecessors and successors of each Tier 1 opcode", + "Top 5 predecessors and successors of each Tier 1 opcode.", iter_pre_succ_pairs_tables, comparative=False, + doc=""" + This does not include the unspecialized instructions that occur after a + specialized instruction deoptimizes. + """, ) def specialization_section() -> Section: def calc_specialization_table(opcode: str) -> RowCalculator: def calc(stats: Stats) -> Rows: + DOCS = { + "deferred": 'Lists the number of "deferred" (i.e. not specialized) instructions executed.', + "hit": "Specialized instructions that complete.", + "miss": "Specialized instructions that deopt.", + "deopt": "Specialized instructions that deopt.", + } + opcode_stats = stats.get_opcode_stats("opcode") total = opcode_stats.get_specialization_total(opcode) specialization_counts = opcode_stats.get_specialization_counts(opcode) return [ ( - f"{label:>12}", + Doc(label, DOCS[label]), Count(count), Ratio(count, total), ) @@ -848,7 +869,7 @@ def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = Non JoinMode.CHANGE, ), Table( - ("", "Count:", "Ratio:"), + ("Success", "Count:", "Ratio:"), calc_specialization_success_failure_table(opcode), JoinMode.CHANGE, ), @@ -862,7 +883,7 @@ def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = Non return Section( "Specialization stats", - "specialization stats by family", + "Specialization stats by family", iter_specialization_tables, ) @@ -880,19 +901,35 @@ def calc_specialization_effectiveness_table(stats: Stats) -> Rows: ) = opcode_stats.get_specialized_total_counts() return [ - ("Basic", Count(basic), Ratio(basic, total)), ( - "Not specialized", + Doc( + "Basic", + "Instructions that are not and cannot be specialized, e.g. `LOAD_FAST`.", + ), + Count(basic), + Ratio(basic, total), + ), + ( + Doc( + "Not specialized", + "Instructions that could be specialized but aren't, e.g. `LOAD_ATTR`, `BINARY_SLICE`.", + ), Count(not_specialized), Ratio(not_specialized, total), ), ( - "Specialized hits", + Doc( + "Specialized hits", + "Specialized instructions, e.g. `LOAD_ATTR_MODULE` that complete.", + ), Count(specialized_hits), Ratio(specialized_hits, total), ), ( - "Specialized misses", + Doc( + "Specialized misses", + "Specialized instructions, e.g. `LOAD_ATTR_MODULE` that deopt.", + ), Count(specialized_misses), Ratio(specialized_misses, total), ), @@ -937,7 +974,7 @@ def calc_misses_by_table(stats: Stats) -> Rows: ), Section( "Deferred by instruction", - "", + "Breakdown of deferred (not specialized) instruction counts by family", [ Table( ("Name", "Count:", "Ratio:"), @@ -948,7 +985,7 @@ def calc_misses_by_table(stats: Stats) -> Rows: ), Section( "Misses by instruction", - "", + "Breakdown of misses (specialized deopts) instruction counts by family", [ Table( ("Name", "Count:", "Ratio:"), @@ -958,6 +995,10 @@ def calc_misses_by_table(stats: Stats) -> Rows: ], ), ], + doc=""" + All entries are execution counts. Should add up to the total number of + Tier 1 instructions executed. + """, ) @@ -980,6 +1021,13 @@ def calc_call_stats_table(stats: Stats) -> Rows: JoinMode.CHANGE, ) ], + doc=""" + This shows what fraction of calls to Python functions are inlined (i.e. + not having a call at the C level) and for those that are not, where the + call comes from. The various categories overlap. + + Also includes the count of frame objects created. + """, ) @@ -993,7 +1041,7 @@ def calc_object_stats_table(stats: Stats) -> Rows: return Section( "Object stats", - "allocations, frees and dict materializatons", + "Allocations, frees and dict materializatons", [ Table( ("", "Count:", "Ratio:"), @@ -1001,6 +1049,16 @@ def calc_object_stats_table(stats: Stats) -> Rows: JoinMode.CHANGE, ) ], + doc=""" + Below, "allocations" means "allocations that are not from a freelist". + Total allocations = "Allocations from freelist" + "Allocations". + + "New values" is the number of values arrays created for objects with + managed dicts. + + The cache hit/miss numbers are for the MRO cache, split into dunder and + other names. + """, ) @@ -1027,6 +1085,9 @@ def calc_gc_stats(stats: Stats) -> Rows: calc_gc_stats, ) ], + doc=""" + Collected/visits gives some measure of efficiency. + """, ) @@ -1131,19 +1192,20 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) def rare_event_section() -> Section: - RARE_DOCS = { - "set class": "Setting an object's class, `obj.__class__ = ...`", - "set bases": "Setting the bases of a class, `cls.__bases__ = ...`", - "set eval frame func": ( - "Setting the PEP 523 frame eval function " - "`_PyInterpreterState_SetFrameEvalFunc()`" - ), - "builtin dict": "Modifying the builtins, `__builtins__.__dict__[var] = ...`", - "func modification": "Modifying a function, e.g. `func.__defaults__ = ...`, etc.", - } - def calc_rare_event_table(stats: Stats) -> Table: - return [(Doc(x, RARE_DOCS[x]), Count(y)) for x, y in stats.get_rare_events()] + DOCS = { + "set class": "Setting an object's class, `obj.__class__ = ...`", + "set bases": "Setting the bases of a class, `cls.__bases__ = ...`", + "set eval frame func": ( + "Setting the PEP 523 frame eval function " + "`_PyInterpreterState_SetFrameEvalFunc()`" + ), + "builtin dict": "Modifying the builtins, `__builtins__.__dict__[var] = ...`", + "func modification": "Modifying a function, e.g. `func.__defaults__ = ...`, etc.", + "watched dict modification": "A watched dict has been modified", + "watched globals modification": "A watched `globals()` dict has been modified", + } + return [(Doc(x, DOCS[x]), Count(y)) for x, y in stats.get_rare_events()] return Section( "Rare events", @@ -1221,24 +1283,36 @@ def to_markdown(x): if len(rows) == 0: return - width = len(header) - header_line = "|" - under_line = "|" + alignments = [] for item in header: - under = "---" + if item.endswith(":"): + alignments.append("right") + else: + alignments.append("left") + + print("", file=out) + print("", file=out) + print("", file=out) + for item, align in zip(header, alignments): if item.endswith(":"): item = item[:-1] - under += ":" - header_line += item + " | " - under_line += under + "|" - print(header_line, file=out) - print(under_line, file=out) + print(f'', file=out) + print("", file=out) + print("", file=out) + + print("", file=out) for row in rows: - if len(row) != width: + if len(row) != len(header): raise ValueError( "Wrong number of elements in row '" + str(row) + "'" ) - print("|", " | ".join(to_markdown(i) for i in row), "|", file=out) + print("", file=out) + for col, align in zip(row, alignments): + print(f'', file=out) + print("", file=out) + print("", file=out) + + print("
{item}
{to_markdown(col)}
", file=out) print(file=out) case list(): From 98cd6743cb7aae78b8e6e288b8158ae3b7f42d89 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 16 Feb 2024 11:22:57 -0500 Subject: [PATCH 4/5] Use dataclasses --- Tools/scripts/summarize_stats.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 68bb2b46bb0a03..ccf875ef75132c 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -11,6 +11,7 @@ import argparse import collections from collections.abc import KeysView +from dataclasses import dataclass from datetime import date import enum import functools @@ -116,10 +117,10 @@ def save_raw_data(data: RawData, json_output: TextIO): json.dump(data, json_output) +@dataclass(frozen=True) class Doc: - def __init__(self, text: str, doc: str): - self.text = text - self.doc = doc + text: str + doc: str def markdown(self) -> str: return textwrap.dedent( @@ -139,11 +140,11 @@ def markdown(self) -> str: return format(self, ",d") +@dataclass(frozen=True) class Ratio: - def __init__(self, num: int, den: int | None, percentage: bool = True): - self.num = num - self.den = den - self.percentage = percentage + num: int + den: int | None = None + percentage: bool = True def __float__(self): if self.den == 0: From ce870476cc17994365bb60bc43853483ddbf4037 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 16 Feb 2024 11:23:07 -0500 Subject: [PATCH 5/5] Define the term "uop" --- Tools/scripts/summarize_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index ccf875ef75132c..5bc39fceb4b2a1 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -493,7 +493,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: executed, None, ), - Doc("Uops executed", "The total number of uops that were executed"): ( + Doc("Uops executed", "The total number of uops (micro-operations) that were executed"): ( uops, executed, ),