8000 Merge branch 'main' into gh-114847 · barneygale/cpython@ecbb27c · GitHub
[go: up one dir, main page]
Skip to content

Commit ecbb27c

Browse files
barneygalebarneygale
authored
Merge branch 'main' into pythongh-114847 
2 parents d2a0248 + 060a96f commit ecbb27c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+2737
-2463
lines changed

‎Doc/howto/logging-cookbook.rst

Lines changed: 64 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1915,14 +1915,15 @@ In a similar way to the above section, we can implement a listener and handler
19151915
using `pynng <https://pypi.org/project/pynng/>`_, which is a Python binding to
19161916
`NNG <https://nng.nanomsg.org/>`_, billed as a spiritual successor to ZeroMQ.
19171917
The following snippets illustrate -- you can test them in an environment which has
1918-
``pynng`` installed. Juat for variety, we present the listener first.
1918+
``pynng`` installed. Just for variety, we present the listener first.
19191919

19201920

19211921
Subclass ``QueueListener``
19221922
^^^^^^^^^^^^^^^^^^^^^^^^^^
19231923

19241924
.. code-block:: python
19251925
1926+
# listener.py
19261927
import json
19271928
import logging
19281929
import logging.handlers
@@ -1955,7 +1956,7 @@ Subclass ``QueueListener``
19551956
break
19561957
except pynng.Timeout:
19571958
pass
1958-
except pynng.Closed: # sometimes hit when you hit Ctrl-C
1959+
except pynng.Closed: # sometimes happens when you hit Ctrl-C
19591960
break
19601961
if data is None:
19611962
return None
@@ -1988,6 +1989,7 @@ Subclass ``QueueHandler``
19881989

19891990
.. code-block:: python
19901991
1992+
# sender.py
19911993
import json
19921994
import logging
19931995
import logging.handlers
@@ -2015,9 +2017,10 @@ Subclass ``QueueHandler``
20152017
20162018
logging.getLogger('pynng').propagate = False
20172019
handler = NNGSocketHandler(DEFAULT_ADDR)
2020+
# Make sure the process ID is in the output
20182021
logging.basicConfig(level=logging.DEBUG,
20192022
handlers=[logging.StreamHandler(), handler],
2020-
format='%(levelname)-8s %(name)10s %(message)s')
2023+
format='%(levelname)-8s %(name)10s %(process)6s %(message)s')
20212024
levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR,
20222025
logging.CRITICAL)
20232026
logger_names = ('myapp', 'myapp.lib1', 'myapp.lib2')
@@ -2031,7 +2034,64 @@ Subclass ``QueueHandler``
20312034
delay = random.random() * 2 + 0.5
20322035
time.sleep(delay)
20332036
2034-
You can run the above two snippets in separate command shells.
2037+
You can run the above two snippets in separate command shells. If we run the
2038+
listener in one shell and run the sender in two separate shells, we should see
2039+
something like the following. In the first sender shell:
2040+
2041+
.. code-block:: console
2042+
2043+
$ python sender.py
2044+
DEBUG myapp 613 Message no. 1
2045+
WARNING myapp.lib2 613 Message no. 2
2046+
CRITICAL myapp.lib2 613 Message no. 3
2047+
WARNING myapp.lib2 613 Message no. 4
2048+
CRITICAL myapp.lib1 613 Message no. 5
2049+
DEBUG myapp 613 Message no. 6
2050+
CRITICAL myapp.lib1 613 Message no. 7
2051+
INFO myapp.lib1 613 Message no. 8
2052+
(and so on)
2053+
2054+
In the second sender shell:
2055+
2056+
.. code-block:: console
2057+
2058+
$ python sender.py
2059+
INFO myapp.lib2 657 Message no. 1
2060+
CRITICAL myapp.lib2 657 Message no. 2
2061+
CRITICAL myapp 657 Message no. 3
2062+
CRITICAL myapp.lib1 657 Message no. 4
2063+
INFO myapp.lib1 657 Message no. 5
2064+
WARNING myapp.lib2 657 Message no. 6
2065+
CRITICAL myapp 657 Message no. 7
2066+
DEBUG myapp.lib1 657 Message no. 8
2067+
(and so on)
2068+
2069+
In the listener shell:
2070+
2071+
.. code-block:: console
2072+
2073+
$ python listener.py
2074+
Press Ctrl-C to stop.
2075+
DEBUG myapp 613 Message no. 1
2076+
WARNING myapp.lib2 613 Message no. 2
2077+
INFO myapp.lib2 657 Message no. 1
2078+
CRITICAL myapp.lib2 613 Message no. 3
2079+
CRITICAL myapp.lib2 657 Message no. 2
2080+
CRITICAL myapp 657 Message no. 3
2081+
WARNING myapp.lib2 613 Message no. 4
2082+
CRITICAL myapp.lib1 613 Message no. 5
2083+
CRITICAL myapp.lib1 657 Message no. 4
2084+
INFO myapp.lib1 657 Message no. 5
2085+
DEBUG myapp 613 Message no. 6
2086+
WARNING myapp.lib2 657 Message no. 6
2087+
CRITICAL myapp 657 Message no. 7
2088+
CRITICAL myapp.lib1 613 Message no. 7
2089+
INFO myapp.lib1 613 Message no. 8
2090+
DEBUG myapp.lib1 657 Message no. 8
2091+
(and so on)
2092+
2093+
As you can see, the logging from the two sender processes is interleaved in the
2094+
listener's output.
20352095

20362096

20372097
An example dictionary-based configuration

‎Doc/library/logging.rst

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,13 @@ If you run *myapp.py*, you should see this in *myapp.log*:
6363
INFO:mylib:Doing something
6464
INFO:__main__:Finished
6565
66-
The key features of this idiomatic usage is that the majority of code is simply
66+
The key feature of this idiomatic usage is that the majority of code is simply
6767
creating a module level logger with ``getLogger(__name__)``, and using that
68-
logger to do any needed logging. This is concise while allowing downstream code
69-
fine grained control if needed. Logged messages to the module-level logger get
70-
forwarded up to handlers of loggers in higher-level modules, all the way up to
71-
the root logger; for this reason this approach is known as hierarchical logging.
68+
logger to do any needed logging. This is concise, while allowing downstream
69+
code fine-grained control if needed. Logged messages to the module-level logger
70+
get forwarded to handlers of loggers in higher-level modules, all the way up to
71+
the highest-level logger known as the root logger; this approach is known as
72+
hierarchical logging.
7273

7374
For logging to be useful, it needs to be configured: setting the levels and
7475
destinations for each logger, potentially changing how specific modules log,
@@ -82,8 +83,8 @@ The module provides a lot of functionality and flexibility. If you are
8283
unfamiliar with logging, the best way to get to grips with it is to view the
8384
tutorials (**see the links above and on the right**).
8485

85-
The basic classes defined by the module, together with their functions, are
86-
listed below.
86+
The basic classes defined by the module, together with their attributes and
87+
methods, are listed in the sections below.
8788

8889
* Loggers expose the interface that application code directly uses.
8990
* Handlers send the log records (created by loggers) to the appropriate

‎Include/cpython/code.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,16 @@ typedef struct _Py_GlobalMonitors {
2424
uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS];
2525
} _Py_GlobalMonitors;
2626

27+
typedef struct {
28+
union {
29+
struct {
30+
uint16_t backoff : 4;
31+
uint16_t value : 12;
32+
};
33+
uint16_t as_counter; // For printf("%#x", ...)
34+
};
35+
} _Py_BackoffCounter;
36+
2737
/* Each instruction in a code object is a fixed-width value,
2838
* currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG
2939
* opcode allows for larger values but the current limit is 3 uses
@@ -39,6 +49,7 @@ typedef union {
3949
uint8_t code;
4050
uint8_t arg;
4151
} op;
52+
_Py_BackoffCounter counter; // First cache entry of specializable op
4253
} _Py_CODEUNIT;
4354

4455

‎Include/cpython/optimizer.h

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
8989

9090
typedef struct _exit_data {
9191
uint32_t target;
92-
int16_t temperature;
92+
_Py_BackoffCounter temperature;
9393
const struct _PyExecutorObject *executor;
9494
} _PyExitData;
9595

@@ -115,11 +115,6 @@ typedef int (*optimize_func)(
115115
struct _PyOptimizerObject {
116116
PyObject_HEAD
117117
optimize_func optimize;
118-
/* These thresholds are treated as signed so do not exceed INT16_MAX
119-
* Use INT16_MAX to indicate that the optimizer should never be called */
120-
uint16_t resume_threshold;
121-
uint16_t side_threshold;
122-
uint16_t backedge_threshold;
123118
/* Data needed by the optimizer goes here, but is opaque to the VM */
124119
};
125120

@@ -151,14 +146,6 @@ extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_inval
151146
PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void);
152147
PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void);
153148

154-
#define OPTIMIZER_BITS_IN_COUNTER 4
155-
/* Minimum of 16 additional executions before retry */
156-
#define MIN_TIER2_BACKOFF 4
157-
#define MAX_TIER2_BACKOFF (15 - OPTIMIZER_BITS_IN_COUNTER)
158-
#define OPTIMIZER_BITS_MASK ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1)
159-
/* A value <= UINT16_MAX but large enough that when shifted is > UINT16_MAX */
160-
#define OPTIMIZER_UNREACHABLE_THRESHOLD UINT16_MAX
161-
162149
#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3
163150
#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6
164151

‎Include/internal/pycore_backoff.h

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
2+
#ifndef Py_INTERNAL_BACKOFF_H
3+
#define Py_INTERNAL_BACKOFF_H
4+
#ifdef __cplusplus
5+
extern "C" {
6+
#endif
7+
8+
#ifndef Py_BUILD_CORE
9+
# error "this header requires Py_BUILD_CORE define"
10+
#endif
11+
12+
#include <assert.h>
13+
#include <stdbool.h>
14+
#include <stdint.h>
15+
16+
/* 16-bit countdown counters using exponential backoff.
17+
18+
These are used by the adaptive specializer to count down until
19+
it is time to specialize an instruction. If specialization fails
20+
the counter is reset using exponential backoff.
21+
22+
Another use is for the Tier 2 optimizer to decide when to create
23+
a new Tier 2 trace (executor). Again, exponential backoff is used.
24+
25+
The 16-bit counter is structured as a 12-bit unsigned 'value'
26+
and a 4-bit 'backoff' field. When resetting the counter, the
27+
backoff field is incremented (until it reaches a limit) and the
28+
value is set to a bit mask representing the value 2**backoff - 1.
29+
The maximum backoff is 12 (the number of value bits).
30+
31+
There is an exceptional value which must not be updated, 0xFFFF.
32+
*/
33+
34+
#define UNREACHABLE_BACKOFF 0xFFFF
35+
36+
static inline bool
37+
is_unreachable_backoff_counter(_Py_BackoffCounter counter)
38+
{
39+
return counter.as_counter == UNREACHABLE_BACKOFF;
40+
}
41+
42+
static inline _Py_BackoffCounter
43+
make_backoff_counter(uint16_t value, uint16_t backoff)
44+
{
45+
assert(backoff <= 15);
46+
assert(value <= 0xFFF);
47+
return (_Py_BackoffCounter){.value = value, .backoff = backoff};
48+
}
49+
50+
static inline _Py_BackoffCounter
51+
forge_backoff_counter(uint16_t counter)
52+
{
53+
return (_Py_BackoffCounter){.as_counter = counter};
54+
}
55+
56+
static inline _Py_BackoffCounter
57+
restart_backoff_counter(_Py_BackoffCounter counter)
58+
{
59+
assert(!is_unreachable_backoff_counter(counter));
60+
if (counter.backoff < 12) {
61+
return make_backoff_counter((1 << (counter.backoff + 1)) - 1, counter.backoff + 1);
62+
}
63+
else {
64+
return make_backoff_counter((1 << 12) - 1, 12);
65+
}
66+
}
67+
68+
static inline _Py_BackoffCounter
69+
pause_backoff_counter(_Py_BackoffCounter counter)
70+
{
71+
return make_backoff_counter(counter.value | 1, counter.backoff);
72+
}
73+
74+
static inline _Py_BackoffCounter
75+
advance_backoff_counter(_Py_BackoffCounter counter)
76+
{
77+
if (!is_unreachable_backoff_counter(counter)) {
78+
return make_backoff_counter((counter.value - 1) & 0xFFF, counter.backoff);
79+
}
80+
else {
81+
return counter;
82+
}
83+
}
84+
85+
static inline bool
86+
backoff_counter_triggers(_Py_BackoffCounter counter)
87+
{
88+
return counter.value == 0;
89+
}
90+
91+
/* Initial JUMP_BACKWARD counter.
92+
* This determines when we create a trace for a loop.
93+
* Backoff sequence 16, 32, 64, 128, 256, 512, 1024, 2048, 4096. */
94+
#define JUMP_BACKWARD_INITIAL_VALUE 16
95+
#define JUMP_BACKWARD_INITIAL_BACKOFF 4
96+
static inline _Py_BackoffCounter
97+
initial_jump_backoff_counter(void)
98+
{
99+
return make_backoff_counter(JUMP_BACKWARD_INITIAL_VALUE,
100+
JUMP_BACKWARD_INITIAL_BACKOFF);
101+
}
102+
103+
/* Initial exit temperature.
104+
* Must be larger than ADAPTIVE_COOLDOWN_VALUE,
105+
* otherwise when a side exit warms up we may construct
106+
* a new trace before the Tier 1 code has properly re-specialized.
107+
* Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */
108+
#define COLD_EXIT_INITIAL_VALUE 64
109+
#define COLD_EXIT_INITIAL_BACKOFF 6
110+
111+
static inline _Py_BackoffCounter
112+
initial_temperature_backoff_counter(void)
113+
{
114+
return make_backoff_counter(COLD_EXIT_INITIAL_VALUE,
115+
COLD_EXIT_INITIAL_BACKOFF);
116+
}
117+
118+
/* Unreachable backoff counter. */
119+
static inline _Py_BackoffCounter
120+
initial_unreachable_backoff_counter(void)
121+
{
122+
return forge_backoff_counter(UNREACHABLE_BACKOFF);
123+
}
124+
125+
#ifdef __cplusplus
126+
}
127+
#endif
128+
#endif /* !Py_INTERNAL_BACKOFF_H */

0 commit comments

Comments
 (0)
0