[go: up one dir, main page]
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Describes operations that can be performed on software-defined page table
4 * leaf entries. These are abstracted from the hardware page table entries
5 * themselves by the softleaf_t type, see mm_types.h.
6 */
7#ifndef _LINUX_LEAFOPS_H
8#define _LINUX_LEAFOPS_H
9
10#include <linux/mm_types.h>
11#include <linux/swapops.h>
12#include <linux/swap.h>
13
14#ifdef CONFIG_MMU
15
16/* Temporary until swp_entry_t eliminated. */
17#define LEAF_TYPE_SHIFT SWP_TYPE_SHIFT
18
19enum softleaf_type {
20 /* Fundamental types. */
21 SOFTLEAF_NONE,
22 SOFTLEAF_SWAP,
23 /* Migration types. */
24 SOFTLEAF_MIGRATION_READ,
25 SOFTLEAF_MIGRATION_READ_EXCLUSIVE,
26 SOFTLEAF_MIGRATION_WRITE,
27 /* Device types. */
28 SOFTLEAF_DEVICE_PRIVATE_READ,
29 SOFTLEAF_DEVICE_PRIVATE_WRITE,
30 SOFTLEAF_DEVICE_EXCLUSIVE,
31 /* H/W posion types. */
32 SOFTLEAF_HWPOISON,
33 /* Marker types. */
34 SOFTLEAF_MARKER,
35};
36
37/**
38 * softleaf_mk_none() - Create an empty ('none') leaf entry.
39 * Returns: empty leaf entry.
40 */
41static inline softleaf_t softleaf_mk_none(void)
42{
43 return ((softleaf_t) { 0 });
44}
45
46/**
47 * softleaf_from_pte() - Obtain a leaf entry from a PTE entry.
48 * @pte: PTE entry.
49 *
50 * If @pte is present (therefore not a leaf entry) the function returns an empty
51 * leaf entry. Otherwise, it returns a leaf entry.
52 *
53 * Returns: Leaf entry.
54 */
55static inline softleaf_t softleaf_from_pte(pte_t pte)
56{
57 softleaf_t arch_entry;
58
59 if (pte_present(a: pte) || pte_none(pte))
60 return softleaf_mk_none();
61
62 pte = pte_swp_clear_flags(pte);
63 arch_entry = __pte_to_swp_entry(pte);
64
65 /* Temporary until swp_entry_t eliminated. */
66 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
67}
68
69/**
70 * softleaf_to_pte() - Obtain a PTE entry from a leaf entry.
71 * @entry: Leaf entry.
72 *
73 * This generates an architecture-specific PTE entry that can be utilised to
74 * encode the metadata the leaf entry encodes.
75 *
76 * Returns: Architecture-specific PTE entry encoding leaf entry.
77 */
78static inline pte_t softleaf_to_pte(softleaf_t entry)
79{
80 /* Temporary until swp_entry_t eliminated. */
81 return swp_entry_to_pte(entry);
82}
83
84#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
85/**
86 * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry.
87 * @pmd: PMD entry.
88 *
89 * If @pmd is present (therefore not a leaf entry) the function returns an empty
90 * leaf entry. Otherwise, it returns a leaf entry.
91 *
92 * Returns: Leaf entry.
93 */
94static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
95{
96 softleaf_t arch_entry;
97
98 if (pmd_present(pmd) || pmd_none(pmd))
99 return softleaf_mk_none();
100
101 if (pmd_swp_soft_dirty(pmd))
102 pmd = pmd_swp_clear_soft_dirty(pmd);
103 if (pmd_swp_uffd_wp(pmd))
104 pmd = pmd_swp_clear_uffd_wp(pmd);
105 arch_entry = __pmd_to_swp_entry(pmd);
106
107 /* Temporary until swp_entry_t eliminated. */
108 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
109}
110
111#else
112
113static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
114{
115 return softleaf_mk_none();
116}
117
118#endif
119
120/**
121 * softleaf_is_none() - Is the leaf entry empty?
122 * @entry: Leaf entry.
123 *
124 * Empty entries are typically the result of a 'none' page table leaf entry
125 * being converted to a leaf entry.
126 *
127 * Returns: true if the entry is empty, false otherwise.
128 */
129static inline bool softleaf_is_none(softleaf_t entry)
130{
131 return entry.val == 0;
132}
133
134/**
135 * softleaf_type() - Identify the type of leaf entry.
136 * @entry: Leaf entry.
137 *
138 * Returns: the leaf entry type associated with @entry.
139 */
140static inline enum softleaf_type softleaf_type(softleaf_t entry)
141{
142 unsigned int type_num;
143
144 if (softleaf_is_none(entry))
145 return SOFTLEAF_NONE;
146
147 type_num = entry.val >> LEAF_TYPE_SHIFT;
148
149 if (type_num < MAX_SWAPFILES)
150 return SOFTLEAF_SWAP;
151
152 switch (type_num) {
153#ifdef CONFIG_MIGRATION
154 case SWP_MIGRATION_READ:
155 return SOFTLEAF_MIGRATION_READ;
156 case SWP_MIGRATION_READ_EXCLUSIVE:
157 return SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
158 case SWP_MIGRATION_WRITE:
159 return SOFTLEAF_MIGRATION_WRITE;
160#endif
161#ifdef CONFIG_DEVICE_PRIVATE
162 case SWP_DEVICE_WRITE:
163 return SOFTLEAF_DEVICE_PRIVATE_WRITE;
164 case SWP_DEVICE_READ:
165 return SOFTLEAF_DEVICE_PRIVATE_READ;
166 case SWP_DEVICE_EXCLUSIVE:
167 return SOFTLEAF_DEVICE_EXCLUSIVE;
168#endif
169#ifdef CONFIG_MEMORY_FAILURE
170 case SWP_HWPOISON:
171 return SOFTLEAF_HWPOISON;
172#endif
173 case SWP_PTE_MARKER:
174 return SOFTLEAF_MARKER;
175 }
176
177 /* Unknown entry type. */
178 VM_WARN_ON_ONCE(1);
179 return SOFTLEAF_NONE;
180}
181
182/**
183 * softleaf_is_swap() - Is this leaf entry a swap entry?
184 * @entry: Leaf entry.
185 *
186 * Returns: true if the leaf entry is a swap entry, otherwise false.
187 */
188static inline bool softleaf_is_swap(softleaf_t entry)
189{
190 return softleaf_type(entry) == SOFTLEAF_SWAP;
191}
192
193/**
194 * softleaf_is_migration_write() - Is this leaf entry a writable migration entry?
195 * @entry: Leaf entry.
196 *
197 * Returns: true if the leaf entry is a writable migration entry, otherwise
198 * false.
199 */
200static inline bool softleaf_is_migration_write(softleaf_t entry)
201{
202 return softleaf_type(entry) == SOFTLEAF_MIGRATION_WRITE;
203}
204
205/**
206 * softleaf_is_migration_read() - Is this leaf entry a readable migration entry?
207 * @entry: Leaf entry.
208 *
209 * Returns: true if the leaf entry is a readable migration entry, otherwise
210 * false.
211 */
212static inline bool softleaf_is_migration_read(softleaf_t entry)
213{
214 return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ;
215}
216
217/**
218 * softleaf_is_migration_read_exclusive() - Is this leaf entry an exclusive
219 * readable migration entry?
220 * @entry: Leaf entry.
221 *
222 * Returns: true if the leaf entry is an exclusive readable migration entry,
223 * otherwise false.
224 */
225static inline bool softleaf_is_migration_read_exclusive(softleaf_t entry)
226{
227 return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
228}
229
230/**
231 * softleaf_is_migration() - Is this leaf entry a migration entry?
232 * @entry: Leaf entry.
233 *
234 * Returns: true if the leaf entry is a migration entry, otherwise false.
235 */
236static inline bool softleaf_is_migration(softleaf_t entry)
237{
238 switch (softleaf_type(entry)) {
239 case SOFTLEAF_MIGRATION_READ:
240 case SOFTLEAF_MIGRATION_READ_EXCLUSIVE:
241 case SOFTLEAF_MIGRATION_WRITE:
242 return true;
243 default:
244 return false;
245 }
246}
247
248/**
249 * softleaf_is_device_private_write() - Is this leaf entry a device private
250 * writable entry?
251 * @entry: Leaf entry.
252 *
253 * Returns: true if the leaf entry is a device private writable entry, otherwise
254 * false.
255 */
256static inline bool softleaf_is_device_private_write(softleaf_t entry)
257{
258 return softleaf_type(entry) == SOFTLEAF_DEVICE_PRIVATE_WRITE;
259}
260
261/**
262 * softleaf_is_device_private() - Is this leaf entry a device private entry?
263 * @entry: Leaf entry.
264 *
265 * Returns: true if the leaf entry is a device private entry, otherwise false.
266 */
267static inline bool softleaf_is_device_private(softleaf_t entry)
268{
269 switch (softleaf_type(entry)) {
270 case SOFTLEAF_DEVICE_PRIVATE_WRITE:
271 case SOFTLEAF_DEVICE_PRIVATE_READ:
272 return true;
273 default:
274 return false;
275 }
276}
277
278/**
279 * softleaf_is_device_exclusive() - Is this leaf entry a device-exclusive entry?
280 * @entry: Leaf entry.
281 *
282 * Returns: true if the leaf entry is a device-exclusive entry, otherwise false.
283 */
284static inline bool softleaf_is_device_exclusive(softleaf_t entry)
285{
286 return softleaf_type(entry) == SOFTLEAF_DEVICE_EXCLUSIVE;
287}
288
289/**
290 * softleaf_is_hwpoison() - Is this leaf entry a hardware poison entry?
291 * @entry: Leaf entry.
292 *
293 * Returns: true if the leaf entry is a hardware poison entry, otherwise false.
294 */
295static inline bool softleaf_is_hwpoison(softleaf_t entry)
296{
297 return softleaf_type(entry) == SOFTLEAF_HWPOISON;
298}
299
300/**
301 * softleaf_is_marker() - Is this leaf entry a marker?
302 * @entry: Leaf entry.
303 *
304 * Returns: true if the leaf entry is a marker entry, otherwise false.
305 */
306static inline bool softleaf_is_marker(softleaf_t entry)
307{
308 return softleaf_type(entry) == SOFTLEAF_MARKER;
309}
310
311/**
312 * softleaf_to_marker() - Obtain marker associated with leaf entry.
313 * @entry: Leaf entry, softleaf_is_marker(@entry) must return true.
314 *
315 * Returns: Marker associated with the leaf entry.
316 */
317static inline pte_marker softleaf_to_marker(softleaf_t entry)
318{
319 VM_WARN_ON_ONCE(!softleaf_is_marker(entry));
320
321 return swp_offset(entry) & PTE_MARKER_MASK;
322}
323
324/**
325 * softleaf_has_pfn() - Does this leaf entry encode a valid PFN number?
326 * @entry: Leaf entry.
327 *
328 * A pfn swap entry is a special type of swap entry that always has a pfn stored
329 * in the swap offset. They can either be used to represent unaddressable device
330 * memory, to restrict access to a page undergoing migration or to represent a
331 * pfn which has been hwpoisoned and unmapped.
332 *
333 * Returns: true if the leaf entry encodes a PFN, otherwise false.
334 */
335static inline bool softleaf_has_pfn(softleaf_t entry)
336{
337 /* Make sure the swp offset can always store the needed fields. */
338 BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);
339
340 if (softleaf_is_migration(entry))
341 return true;
342 if (softleaf_is_device_private(entry))
343 return true;
344 if (softleaf_is_device_exclusive(entry))
345 return true;
346 if (softleaf_is_hwpoison(entry))
347 return true;
348
349 return false;
350}
351
352/**
353 * softleaf_to_pfn() - Obtain PFN encoded within leaf entry.
354 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
355 *
356 * Returns: The PFN associated with the leaf entry.
357 */
358static inline unsigned long softleaf_to_pfn(softleaf_t entry)
359{
360 VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
361
362 /* Temporary until swp_entry_t eliminated. */
363 return swp_offset(entry) & SWP_PFN_MASK;
364}
365
366/**
367 * softleaf_to_page() - Obtains struct page for PFN encoded within leaf entry.
368 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
369 *
370 * Returns: Pointer to the struct page associated with the leaf entry's PFN.
371 */
372static inline struct page *softleaf_to_page(softleaf_t entry)
373{
374 struct page *page = pfn_to_page(softleaf_to_pfn(entry));
375
376 VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
377 /*
378 * Any use of migration entries may only occur while the
379 * corresponding page is locked
380 */
381 VM_WARN_ON_ONCE(softleaf_is_migration(entry) && !PageLocked(page));
382
383 return page;
384}
385
386/**
387 * softleaf_to_folio() - Obtains struct folio for PFN encoded within leaf entry.
388 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
389 *
390 * Returns: Pointer to the struct folio associated with the leaf entry's PFN.
391 */
392static inline struct folio *softleaf_to_folio(softleaf_t entry)
393{
394 struct folio *folio = pfn_folio(pfn: softleaf_to_pfn(entry));
395
396 VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
397 /*
398 * Any use of migration entries may only occur while the
399 * corresponding folio is locked.
400 */
401 VM_WARN_ON_ONCE(softleaf_is_migration(entry) &&
402 !folio_test_locked(folio));
403
404 return folio;
405}
406
407/**
408 * softleaf_is_poison_marker() - Is this leaf entry a poison marker?
409 * @entry: Leaf entry.
410 *
411 * The poison marker is set via UFFDIO_POISON. Userfaultfd-specific.
412 *
413 * Returns: true if the leaf entry is a poison marker, otherwise false.
414 */
415static inline bool softleaf_is_poison_marker(softleaf_t entry)
416{
417 if (!softleaf_is_marker(entry))
418 return false;
419
420 return softleaf_to_marker(entry) & PTE_MARKER_POISONED;
421}
422
423/**
424 * softleaf_is_guard_marker() - Is this leaf entry a guard region marker?
425 * @entry: Leaf entry.
426 *
427 * Returns: true if the leaf entry is a guard marker, otherwise false.
428 */
429static inline bool softleaf_is_guard_marker(softleaf_t entry)
430{
431 if (!softleaf_is_marker(entry))
432 return false;
433
434 return softleaf_to_marker(entry) & PTE_MARKER_GUARD;
435}
436
437/**
438 * softleaf_is_uffd_wp_marker() - Is this leaf entry a userfautlfd write protect
439 * marker?
440 * @entry: Leaf entry.
441 *
442 * Userfaultfd-specific.
443 *
444 * Returns: true if the leaf entry is a UFFD WP marker, otherwise false.
445 */
446static inline bool softleaf_is_uffd_wp_marker(softleaf_t entry)
447{
448 if (!softleaf_is_marker(entry))
449 return false;
450
451 return softleaf_to_marker(entry) & PTE_MARKER_UFFD_WP;
452}
453
454#ifdef CONFIG_MIGRATION
455
456/**
457 * softleaf_is_migration_young() - Does this migration entry contain an accessed
458 * bit?
459 * @entry: Leaf entry.
460 *
461 * If the architecture can support storing A/D bits in migration entries, this
462 * determines whether the accessed (or 'young') bit was set on the migrated page
463 * table entry.
464 *
465 * Returns: true if the entry contains an accessed bit, otherwise false.
466 */
467static inline bool softleaf_is_migration_young(softleaf_t entry)
468{
469 VM_WARN_ON_ONCE(!softleaf_is_migration(entry));
470
471 if (migration_entry_supports_ad())
472 return swp_offset(entry) & SWP_MIG_YOUNG;
473 /* Keep the old behavior of aging page after migration */
474 return false;
475}
476
477/**
478 * softleaf_is_migration_dirty() - Does this migration entry contain a dirty bit?
479 * @entry: Leaf entry.
480 *
481 * If the architecture can support storing A/D bits in migration entries, this
482 * determines whether the dirty bit was set on the migrated page table entry.
483 *
484 * Returns: true if the entry contains a dirty bit, otherwise false.
485 */
486static inline bool softleaf_is_migration_dirty(softleaf_t entry)
487{
488 VM_WARN_ON_ONCE(!softleaf_is_migration(entry));
489
490 if (migration_entry_supports_ad())
491 return swp_offset(entry) & SWP_MIG_DIRTY;
492 /* Keep the old behavior of clean page after migration */
493 return false;
494}
495
496#else /* CONFIG_MIGRATION */
497
498static inline bool softleaf_is_migration_young(softleaf_t entry)
499{
500 return false;
501}
502
503static inline bool softleaf_is_migration_dirty(softleaf_t entry)
504{
505 return false;
506}
507#endif /* CONFIG_MIGRATION */
508
509/**
510 * pte_is_marker() - Does the PTE entry encode a marker leaf entry?
511 * @pte: PTE entry.
512 *
513 * Returns: true if this PTE is a marker leaf entry, otherwise false.
514 */
515static inline bool pte_is_marker(pte_t pte)
516{
517 return softleaf_is_marker(entry: softleaf_from_pte(pte));
518}
519
520/**
521 * pte_is_uffd_wp_marker() - Does this PTE entry encode a userfaultfd write
522 * protect marker leaf entry?
523 * @pte: PTE entry.
524 *
525 * Returns: true if this PTE is a UFFD WP marker leaf entry, otherwise false.
526 */
527static inline bool pte_is_uffd_wp_marker(pte_t pte)
528{
529 const softleaf_t entry = softleaf_from_pte(pte);
530
531 return softleaf_is_uffd_wp_marker(entry);
532}
533
534/**
535 * pte_is_uffd_marker() - Does this PTE entry encode a userfault-specific marker
536 * leaf entry?
537 * @pte: PTE entry.
538 *
539 * It's useful to be able to determine which leaf entries encode UFFD-specific
540 * markers so we can handle these correctly.
541 *
542 * Returns: true if this PTE entry is a UFFD-specific marker, otherwise false.
543 */
544static inline bool pte_is_uffd_marker(pte_t pte)
545{
546 const softleaf_t entry = softleaf_from_pte(pte);
547
548 if (!softleaf_is_marker(entry))
549 return false;
550
551 /* UFFD WP, poisoned swap entries are UFFD-handled. */
552 if (softleaf_is_uffd_wp_marker(entry))
553 return true;
554 if (softleaf_is_poison_marker(entry))
555 return true;
556
557 return false;
558}
559
560#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION)
561
562/**
563 * pmd_is_device_private_entry() - Check if PMD contains a device private swap
564 * entry.
565 * @pmd: The PMD to check.
566 *
567 * Returns true if the PMD contains a swap entry that represents a device private
568 * page mapping. This is used for zone device private pages that have been
569 * swapped out but still need special handling during various memory management
570 * operations.
571 *
572 * Return: true if PMD contains device private entry, false otherwise
573 */
574static inline bool pmd_is_device_private_entry(pmd_t pmd)
575{
576 return softleaf_is_device_private(entry: softleaf_from_pmd(pmd));
577}
578
579#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
580
581static inline bool pmd_is_device_private_entry(pmd_t pmd)
582{
583 return false;
584}
585
586#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
587
588/**
589 * pmd_is_migration_entry() - Does this PMD entry encode a migration entry?
590 * @pmd: PMD entry.
591 *
592 * Returns: true if the PMD encodes a migration entry, otherwise false.
593 */
594static inline bool pmd_is_migration_entry(pmd_t pmd)
595{
596 return softleaf_is_migration(entry: softleaf_from_pmd(pmd));
597}
598
599/**
600 * pmd_is_valid_softleaf() - Is this PMD entry a valid leaf entry?
601 * @pmd: PMD entry.
602 *
603 * PMD leaf entries are valid only if they are device private or migration
604 * entries. This function asserts that a PMD leaf entry is valid in this
605 * respect.
606 *
607 * Returns: true if the PMD entry is a valid leaf entry, otherwise false.
608 */
609static inline bool pmd_is_valid_softleaf(pmd_t pmd)
610{
611 const softleaf_t entry = softleaf_from_pmd(pmd);
612
613 /* Only device private, migration entries valid for PMD. */
614 return softleaf_is_device_private(entry) ||
615 softleaf_is_migration(entry);
616}
617
618#endif /* CONFIG_MMU */
619#endif /* _LINUX_LEAFOPS_H */
620

source code of linux/include/linux/leafops.h