[go: up one dir, main page]
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * NUMA memory policies for Linux.
4 * Copyright 2003,2004 Andi Kleen SuSE Labs
5 */
6#ifndef _LINUX_MEMPOLICY_H
7#define _LINUX_MEMPOLICY_H 1
8
9#include <linux/sched.h>
10#include <linux/mmzone.h>
11#include <linux/slab.h>
12#include <linux/rbtree.h>
13#include <linux/spinlock.h>
14#include <linux/node.h>
15#include <linux/nodemask.h>
16#include <linux/pagemap.h>
17#include <uapi/linux/mempolicy.h>
18
19struct mm_struct;
20
21#define NO_INTERLEAVE_INDEX (-1UL) /* use task il_prev for interleaving */
22
23#ifdef CONFIG_NUMA
24
25/*
26 * Describe a memory policy.
27 *
28 * A mempolicy can be either associated with a process or with a VMA.
29 * For VMA related allocations the VMA policy is preferred, otherwise
30 * the process policy is used. Interrupts ignore the memory policy
31 * of the current process.
32 *
33 * Locking policy for interleave:
34 * In process context there is no locking because only the process accesses
35 * its own state. All vma manipulation is somewhat protected by a down_read on
36 * mmap_lock.
37 *
38 * Freeing policy:
39 * Mempolicy objects are reference counted. A mempolicy will be freed when
40 * mpol_put() decrements the reference count to zero.
41 *
42 * Duplicating policy objects:
43 * mpol_dup() allocates a new mempolicy and copies the specified mempolicy
44 * to the new storage. The reference count of the new object is initialized
45 * to 1, representing the caller of mpol_dup().
46 */
47struct mempolicy {
48 atomic_t refcnt;
49 unsigned short mode; /* See MPOL_* above */
50 unsigned short flags; /* See set_mempolicy() MPOL_F_* above */
51 nodemask_t nodes; /* interleave/bind/preferred/etc */
52 int home_node; /* Home node to use for MPOL_BIND and MPOL_PREFERRED_MANY */
53
54 union {
55 nodemask_t cpuset_mems_allowed; /* relative to these nodes */
56 nodemask_t user_nodemask; /* nodemask passed by user */
57 } w;
58};
59
60/*
61 * Support for managing mempolicy data objects (clone, copy, destroy)
62 * The default fast path of a NULL MPOL_DEFAULT policy is always inlined.
63 */
64
65extern void __mpol_put(struct mempolicy *pol);
66static inline void mpol_put(struct mempolicy *pol)
67{
68 if (pol)
69 __mpol_put(pol);
70}
71
72/*
73 * Does mempolicy pol need explicit unref after use?
74 * Currently only needed for shared policies.
75 */
76static inline int mpol_needs_cond_ref(struct mempolicy *pol)
77{
78 return (pol && (pol->flags & MPOL_F_SHARED));
79}
80
81static inline void mpol_cond_put(struct mempolicy *pol)
82{
83 if (mpol_needs_cond_ref(pol))
84 __mpol_put(pol);
85}
86
87extern struct mempolicy *__mpol_dup(struct mempolicy *pol);
88static inline struct mempolicy *mpol_dup(struct mempolicy *pol)
89{
90 if (pol)
91 pol = __mpol_dup(pol);
92 return pol;
93}
94
95static inline void mpol_get(struct mempolicy *pol)
96{
97 if (pol)
98 atomic_inc(v: &pol->refcnt);
99}
100
101extern bool __mpol_equal(struct mempolicy *a, struct mempolicy *b);
102static inline bool mpol_equal(struct mempolicy *a, struct mempolicy *b)
103{
104 if (a == b)
105 return true;
106 return __mpol_equal(a, b);
107}
108
109/*
110 * Tree of shared policies for a shared memory region.
111 */
112struct shared_policy {
113 struct rb_root root;
114 rwlock_t lock;
115};
116struct sp_node {
117 struct rb_node nd;
118 pgoff_t start, end;
119 struct mempolicy *policy;
120};
121
122int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst);
123void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol);
124int mpol_set_shared_policy(struct shared_policy *sp,
125 struct vm_area_struct *vma, struct mempolicy *mpol);
126void mpol_free_shared_policy(struct shared_policy *sp);
127struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp,
128 pgoff_t idx);
129
130struct mempolicy *get_task_policy(struct task_struct *p);
131struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
132 unsigned long addr, pgoff_t *ilx);
133struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
134 unsigned long addr, int order, pgoff_t *ilx);
135bool vma_policy_mof(struct vm_area_struct *vma);
136
137extern void numa_default_policy(void);
138extern void numa_policy_init(void);
139extern void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new);
140extern void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new);
141
142extern int huge_node(struct vm_area_struct *vma,
143 unsigned long addr, gfp_t gfp_flags,
144 struct mempolicy **mpol, nodemask_t **nodemask);
145extern bool init_nodemask_of_mempolicy(nodemask_t *mask);
146extern bool mempolicy_in_oom_domain(struct task_struct *tsk,
147 const nodemask_t *mask);
148extern unsigned int mempolicy_slab_node(void);
149
150extern enum zone_type policy_zone;
151
152static inline void check_highest_zone(enum zone_type k)
153{
154 if (k > policy_zone && k != ZONE_MOVABLE)
155 policy_zone = k;
156}
157
158int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
159 const nodemask_t *to, int flags);
160
161
162#ifdef CONFIG_TMPFS
163extern int mpol_parse_str(char *str, struct mempolicy **mpol);
164#endif
165
166extern void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol);
167
168/* Check if a vma is migratable */
169extern bool vma_migratable(struct vm_area_struct *vma);
170
171int mpol_misplaced(struct folio *folio, struct vm_fault *vmf,
172 unsigned long addr);
173extern void mpol_put_task_policy(struct task_struct *);
174
175static inline bool mpol_is_preferred_many(struct mempolicy *pol)
176{
177 return (pol->mode == MPOL_PREFERRED_MANY);
178}
179
180extern bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone);
181
182extern int mempolicy_set_node_perf(unsigned int node,
183 struct access_coordinate *coords);
184
185#else
186
187struct mempolicy {};
188
189static inline struct mempolicy *get_task_policy(struct task_struct *p)
190{
191 return NULL;
192}
193
194static inline bool mpol_equal(struct mempolicy *a, struct mempolicy *b)
195{
196 return true;
197}
198
199static inline void mpol_put(struct mempolicy *pol)
200{
201}
202
203static inline void mpol_cond_put(struct mempolicy *pol)
204{
205}
206
207static inline void mpol_get(struct mempolicy *pol)
208{
209}
210
211struct shared_policy {};
212
213static inline void mpol_shared_policy_init(struct shared_policy *sp,
214 struct mempolicy *mpol)
215{
216}
217
218static inline void mpol_free_shared_policy(struct shared_policy *sp)
219{
220}
221
222static inline struct mempolicy *
223mpol_shared_policy_lookup(struct shared_policy *sp, pgoff_t idx)
224{
225 return NULL;
226}
227
228static inline struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
229 unsigned long addr, int order, pgoff_t *ilx)
230{
231 *ilx = 0;
232 return NULL;
233}
234
235static inline int
236vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
237{
238 return 0;
239}
240
241static inline void numa_policy_init(void)
242{
243}
244
245static inline void numa_default_policy(void)
246{
247}
248
249static inline void mpol_rebind_task(struct task_struct *tsk,
250 const nodemask_t *new)
251{
252}
253
254static inline void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
255{
256}
257
258static inline int huge_node(struct vm_area_struct *vma,
259 unsigned long addr, gfp_t gfp_flags,
260 struct mempolicy **mpol, nodemask_t **nodemask)
261{
262 *mpol = NULL;
263 *nodemask = NULL;
264 return 0;
265}
266
267static inline bool init_nodemask_of_mempolicy(nodemask_t *m)
268{
269 return false;
270}
271
272static inline int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
273 const nodemask_t *to, int flags)
274{
275 return 0;
276}
277
278static inline void check_highest_zone(int k)
279{
280}
281
282#ifdef CONFIG_TMPFS
283static inline int mpol_parse_str(char *str, struct mempolicy **mpol)
284{
285 return 1; /* error */
286}
287#endif
288
289static inline int mpol_misplaced(struct folio *folio,
290 struct vm_fault *vmf,
291 unsigned long address)
292{
293 return -1; /* no node preference */
294}
295
296static inline void mpol_put_task_policy(struct task_struct *task)
297{
298}
299
300static inline bool mpol_is_preferred_many(struct mempolicy *pol)
301{
302 return false;
303}
304
305#endif /* CONFIG_NUMA */
306#endif
307

source code of linux/include/linux/mempolicy.h