1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Framework for buffer objects that can be shared across devices/subsystems.
4	*
5	* Copyright(C) 2011 Linaro Limited. All rights reserved.
6	* Author: Sumit Semwal <sumit.semwal@ti.com>
7	*
8	* Many thanks to linaro-mm-sig list, and specially
9	* Arnd Bergmann <arnd@arndb.de>, Rob Clark <rob@ti.com> and
10	* Daniel Vetter <daniel@ffwll.ch> for their support in creation and
11	* refining of this idea.
12	*/
13
14	#include <linux/fs.h>
15	#include <linux/slab.h>
16	#include <linux/dma-buf.h>
17	#include <linux/dma-fence.h>
18	#include <linux/dma-fence-unwrap.h>
19	#include <linux/anon_inodes.h>
20	#include <linux/export.h>
21	#include <linux/debugfs.h>
22	#include <linux/list.h>
23	#include <linux/module.h>
24	#include <linux/mutex.h>
25	#include <linux/seq_file.h>
26	#include <linux/sync_file.h>
27	#include <linux/poll.h>
28	#include <linux/dma-resv.h>
29	#include <linux/mm.h>
30	#include <linux/mount.h>
31	#include <linux/pseudo_fs.h>
32
33	#include <uapi/linux/dma-buf.h>
34	#include <uapi/linux/magic.h>
35
36	#include "dma-buf-sysfs-stats.h"
37
38	static inline int is_dma_buf_file(struct file *);
39
40	static DEFINE_MUTEX(dmabuf_list_mutex);
41	static LIST_HEAD(dmabuf_list);
42
43	static void __dma_buf_list_add(struct dma_buf *dmabuf)
44	{
45	mutex_lock(&dmabuf_list_mutex);
46	list_add(new: &dmabuf->list_node, head: &dmabuf_list);
47	mutex_unlock(lock: &dmabuf_list_mutex);
48	}
49
50	static void __dma_buf_list_del(struct dma_buf *dmabuf)
51	{
52	if (!dmabuf)
53	return;
54
55	mutex_lock(&dmabuf_list_mutex);
56	list_del(entry: &dmabuf->list_node);
57	mutex_unlock(lock: &dmabuf_list_mutex);
58	}
59
60	/**
61	* dma_buf_iter_begin - begin iteration through global list of all DMA buffers
62	*
63	* Returns the first buffer in the global list of DMA-bufs that's not in the
64	* process of being destroyed. Increments that buffer's reference count to
65	* prevent buffer destruction. Callers must release the reference, either by
66	* continuing iteration with dma_buf_iter_next(), or with dma_buf_put().
67	*
68	* Return:
69	* * First buffer from global list, with refcount elevated
70	* * NULL if no active buffers are present
71	*/
72	struct dma_buf *dma_buf_iter_begin(void)
73	{
74	struct dma_buf *ret = NULL, *dmabuf;
75
76	/*
77	* The list mutex does not protect a dmabuf's refcount, so it can be
78	* zeroed while we are iterating. We cannot call get_dma_buf() since the
79	* caller may not already own a reference to the buffer.
80	*/
81	mutex_lock(&dmabuf_list_mutex);
82	list_for_each_entry(dmabuf, &dmabuf_list, list_node) {
83	if (file_ref_get(ref: &dmabuf->file->f_ref)) {
84	ret = dmabuf;
85	break;
86	}
87	}
88	mutex_unlock(lock: &dmabuf_list_mutex);
89	return ret;
90	}
91
92	/**
93	* dma_buf_iter_next - continue iteration through global list of all DMA buffers
94	* @dmabuf: [in] pointer to dma_buf
95	*
96	* Decrements the reference count on the provided buffer. Returns the next
97	* buffer from the remainder of the global list of DMA-bufs with its reference
98	* count incremented. Callers must release the reference, either by continuing
99	* iteration with dma_buf_iter_next(), or with dma_buf_put().
100	*
101	* Return:
102	* * Next buffer from global list, with refcount elevated
103	* * NULL if no additional active buffers are present
104	*/
105	struct dma_buf *dma_buf_iter_next(struct dma_buf *dmabuf)
106	{
107	struct dma_buf *ret = NULL;
108
109	/*
110	* The list mutex does not protect a dmabuf's refcount, so it can be
111	* zeroed while we are iterating. We cannot call get_dma_buf() since the
112	* caller may not already own a reference to the buffer.
113	*/
114	mutex_lock(&dmabuf_list_mutex);
115	dma_buf_put(dmabuf);
116	list_for_each_entry_continue(dmabuf, &dmabuf_list, list_node) {
117	if (file_ref_get(ref: &dmabuf->file->f_ref)) {
118	ret = dmabuf;
119	break;
120	}
121	}
122	mutex_unlock(lock: &dmabuf_list_mutex);
123	return ret;
124	}
125
126	static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen)
127	{
128	struct dma_buf *dmabuf;
129	char name[DMA_BUF_NAME_LEN];
130	ssize_t ret = 0;
131
132	dmabuf = dentry->d_fsdata;
133	spin_lock(lock: &dmabuf->name_lock);
134	if (dmabuf->name)
135	ret = strscpy(name, dmabuf->name, sizeof(name));
136	spin_unlock(lock: &dmabuf->name_lock);
137
138	return dynamic_dname(buffer, buflen, "/%s:%s",
139	dentry->d_name.name, ret > 0 ? name : "");
140	}
141
142	static void dma_buf_release(struct dentry *dentry)
143	{
144	struct dma_buf *dmabuf;
145
146	dmabuf = dentry->d_fsdata;
147	if (unlikely(!dmabuf))
148	return;
149
150	BUG_ON(dmabuf->vmapping_counter);
151
152	/*
153	* If you hit this BUG() it could mean:
154	* * There's a file reference imbalance in dma_buf_poll / dma_buf_poll_cb or somewhere else
155	* * dmabuf->cb_in/out.active are non-0 despite no pending fence callback
156	*/
157	BUG_ON(dmabuf->cb_in.active || dmabuf->cb_out.active);
158
159	dma_buf_stats_teardown(dmabuf);
160	dmabuf->ops->release(dmabuf);
161
162	if (dmabuf->resv == (struct dma_resv *)&dmabuf[1])
163	dma_resv_fini(obj: dmabuf->resv);
164
165	WARN_ON(!list_empty(&dmabuf->attachments));
166	module_put(module: dmabuf->owner);
167	kfree(objp: dmabuf->name);
168	kfree(objp: dmabuf);
169	}
170
171	static int dma_buf_file_release(struct inode *inode, struct file *file)
172	{
173	if (!is_dma_buf_file(file))
174	return -EINVAL;
175
176	__dma_buf_list_del(dmabuf: file->private_data);
177
178	return 0;
179	}
180
181	static const struct dentry_operations dma_buf_dentry_ops = {
182	.d_dname = dmabuffs_dname,
183	.d_release = dma_buf_release,
184	};
185
186	static struct vfsmount *dma_buf_mnt;
187
188	static int dma_buf_fs_init_context(struct fs_context *fc)
189	{
190	struct pseudo_fs_context *ctx;
191
192	ctx = init_pseudo(fc, DMA_BUF_MAGIC);
193	if (!ctx)
194	return -ENOMEM;
195	ctx->dops = &dma_buf_dentry_ops;
196	return 0;
197	}
198
199	static struct file_system_type dma_buf_fs_type = {
200	.name = "dmabuf",
201	.init_fs_context = dma_buf_fs_init_context,
202	.kill_sb = kill_anon_super,
203	};
204
205	static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma)
206	{
207	struct dma_buf *dmabuf;
208
209	if (!is_dma_buf_file(file))
210	return -EINVAL;
211
212	dmabuf = file->private_data;
213
214	/* check if buffer supports mmap */
215	if (!dmabuf->ops->mmap)
216	return -EINVAL;
217
218	/* check for overflowing the buffer's size */
219	if (vma->vm_pgoff + vma_pages(vma) >
220	dmabuf->size >> PAGE_SHIFT)
221	return -EINVAL;
222
223	return dmabuf->ops->mmap(dmabuf, vma);
224	}
225
226	static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence)
227	{
228	struct dma_buf *dmabuf;
229	loff_t base;
230
231	if (!is_dma_buf_file(file))
232	return -EBADF;
233
234	dmabuf = file->private_data;
235
236	/* only support discovering the end of the buffer,
237	* but also allow SEEK_SET to maintain the idiomatic
238	* SEEK_END(0), SEEK_CUR(0) pattern.
239	*/
240	if (whence == SEEK_END)
241	base = dmabuf->size;
242	else if (whence == SEEK_SET)
243	base = 0;
244	else
245	return -EINVAL;
246
247	if (offset != 0)
248	return -EINVAL;
249
250	return base + offset;
251	}
252
253	/**
254	* DOC: implicit fence polling
255	*
256	* To support cross-device and cross-driver synchronization of buffer access
257	* implicit fences (represented internally in the kernel with &struct dma_fence)
258	* can be attached to a &dma_buf. The glue for that and a few related things are
259	* provided in the &dma_resv structure.
260	*
261	* Userspace can query the state of these implicitly tracked fences using poll()
262	* and related system calls:
263	*
264	* - Checking for EPOLLIN, i.e. read access, can be use to query the state of the
265	* most recent write or exclusive fence.
266	*
267	* - Checking for EPOLLOUT, i.e. write access, can be used to query the state of
268	* all attached fences, shared and exclusive ones.
269	*
270	* Note that this only signals the completion of the respective fences, i.e. the
271	* DMA transfers are complete. Cache flushing and any other necessary
272	* preparations before CPU access can begin still need to happen.
273	*
274	* As an alternative to poll(), the set of fences on DMA buffer can be
275	* exported as a &sync_file using &dma_buf_sync_file_export.
276	*/
277
278	static void dma_buf_poll_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
279	{
280	struct dma_buf_poll_cb_t *dcb = (struct dma_buf_poll_cb_t *)cb;
281	struct dma_buf *dmabuf = container_of(dcb->poll, struct dma_buf, poll);
282	unsigned long flags;
283
284	spin_lock_irqsave(&dcb->poll->lock, flags);
285	wake_up_locked_poll(dcb->poll, dcb->active);
286	dcb->active = 0;
287	spin_unlock_irqrestore(lock: &dcb->poll->lock, flags);
288	dma_fence_put(fence);
289	/* Paired with get_file in dma_buf_poll */
290	fput(dmabuf->file);
291	}
292
293	static bool dma_buf_poll_add_cb(struct dma_resv *resv, bool write,
294	struct dma_buf_poll_cb_t *dcb)
295	{
296	struct dma_resv_iter cursor;
297	struct dma_fence *fence;
298	int r;
299
300	dma_resv_for_each_fence(&cursor, resv, dma_resv_usage_rw(write),
301	fence) {
302	dma_fence_get(fence);
303	r = dma_fence_add_callback(fence, cb: &dcb->cb, func: dma_buf_poll_cb);
304	if (!r)
305	return true;
306	dma_fence_put(fence);
307	}
308
309	return false;
310	}
311
312	static __poll_t dma_buf_poll(struct file *file, poll_table *poll)
313	{
314	struct dma_buf *dmabuf;
315	struct dma_resv *resv;
316	__poll_t events;
317
318	dmabuf = file->private_data;
319	if (!dmabuf || !dmabuf->resv)
320	return EPOLLERR;
321
322	resv = dmabuf->resv;
323
324	poll_wait(filp: file, wait_address: &dmabuf->poll, p: poll);
325
326	events = poll_requested_events(p: poll) & (EPOLLIN | EPOLLOUT);
327	if (!events)
328	return 0;
329
330	dma_resv_lock(obj: resv, NULL);
331
332	if (events & EPOLLOUT) {
333	struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_out;
334
335	/* Check that callback isn't busy */
336	spin_lock_irq(lock: &dmabuf->poll.lock);
337	if (dcb->active)
338	events &= ~EPOLLOUT;
339	else
340	dcb->active = EPOLLOUT;
341	spin_unlock_irq(lock: &dmabuf->poll.lock);
342
343	if (events & EPOLLOUT) {
344	/* Paired with fput in dma_buf_poll_cb */
345	get_file(f: dmabuf->file);
346
347	if (!dma_buf_poll_add_cb(resv, write: true, dcb))
348	/* No callback queued, wake up any other waiters */
349	dma_buf_poll_cb(NULL, cb: &dcb->cb);
350	else
351	events &= ~EPOLLOUT;
352	}
353	}
354
355	if (events & EPOLLIN) {
356	struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_in;
357
358	/* Check that callback isn't busy */
359	spin_lock_irq(lock: &dmabuf->poll.lock);
360	if (dcb->active)
361	events &= ~EPOLLIN;
362	else
363	dcb->active = EPOLLIN;
364	spin_unlock_irq(lock: &dmabuf->poll.lock);
365
366	if (events & EPOLLIN) {
367	/* Paired with fput in dma_buf_poll_cb */
368	get_file(f: dmabuf->file);
369
370	if (!dma_buf_poll_add_cb(resv, write: false, dcb))
371	/* No callback queued, wake up any other waiters */
372	dma_buf_poll_cb(NULL, cb: &dcb->cb);
373	else
374	events &= ~EPOLLIN;
375	}
376	}
377
378	dma_resv_unlock(obj: resv);
379	return events;
380	}
381
382	/**
383	* dma_buf_set_name - Set a name to a specific dma_buf to track the usage.
384	* It could support changing the name of the dma-buf if the same
385	* piece of memory is used for multiple purpose between different devices.
386	*
387	* @dmabuf: [in] dmabuf buffer that will be renamed.
388	* @buf: [in] A piece of userspace memory that contains the name of
389	* the dma-buf.
390	*
391	* Returns 0 on success. If the dma-buf buffer is already attached to
392	* devices, return -EBUSY.
393	*
394	*/
395	static long dma_buf_set_name(struct dma_buf *dmabuf, const char __user *buf)
396	{
397	char *name = strndup_user(buf, DMA_BUF_NAME_LEN);
398
399	if (IS_ERR(ptr: name))
400	return PTR_ERR(ptr: name);
401
402	spin_lock(lock: &dmabuf->name_lock);
403	kfree(objp: dmabuf->name);
404	dmabuf->name = name;
405	spin_unlock(lock: &dmabuf->name_lock);
406
407	return 0;
408	}
409
410	#if IS_ENABLED(CONFIG_SYNC_FILE)
411	static long dma_buf_export_sync_file(struct dma_buf *dmabuf,
412	void __user *user_data)
413	{
414	struct dma_buf_export_sync_file arg;
415	enum dma_resv_usage usage;
416	struct dma_fence *fence = NULL;
417	struct sync_file *sync_file;
418	int fd, ret;
419
420	if (copy_from_user(to: &arg, from: user_data, n: sizeof(arg)))
421	return -EFAULT;
422
423	if (arg.flags & ~DMA_BUF_SYNC_RW)
424	return -EINVAL;
425
426	if ((arg.flags & DMA_BUF_SYNC_RW) == 0)
427	return -EINVAL;
428
429	fd = get_unused_fd_flags(O_CLOEXEC);
430	if (fd < 0)
431	return fd;
432
433	usage = dma_resv_usage_rw(write: arg.flags & DMA_BUF_SYNC_WRITE);
434	ret = dma_resv_get_singleton(obj: dmabuf->resv, usage, fence: &fence);
435	if (ret)
436	goto err_put_fd;
437
438	if (!fence)
439	fence = dma_fence_get_stub();
440
441	sync_file = sync_file_create(fence);
442
443	dma_fence_put(fence);
444
445	if (!sync_file) {
446	ret = -ENOMEM;
447	goto err_put_fd;
448	}
449
450	arg.fd = fd;
451	if (copy_to_user(to: user_data, from: &arg, n: sizeof(arg))) {
452	ret = -EFAULT;
453	goto err_put_file;
454	}
455
456	fd_install(fd, file: sync_file->file);
457
458	return 0;
459
460	err_put_file:
461	fput(sync_file->file);
462	err_put_fd:
463	put_unused_fd(fd);
464	return ret;
465	}
466
467	static long dma_buf_import_sync_file(struct dma_buf *dmabuf,
468	const void __user *user_data)
469	{
470	struct dma_buf_import_sync_file arg;
471	struct dma_fence *fence, *f;
472	enum dma_resv_usage usage;
473	struct dma_fence_unwrap iter;
474	unsigned int num_fences;
475	int ret = 0;
476
477	if (copy_from_user(to: &arg, from: user_data, n: sizeof(arg)))
478	return -EFAULT;
479
480	if (arg.flags & ~DMA_BUF_SYNC_RW)
481	return -EINVAL;
482
483	if ((arg.flags & DMA_BUF_SYNC_RW) == 0)
484	return -EINVAL;
485
486	fence = sync_file_get_fence(fd: arg.fd);
487	if (!fence)
488	return -EINVAL;
489
490	usage = (arg.flags & DMA_BUF_SYNC_WRITE) ? DMA_RESV_USAGE_WRITE :
491	DMA_RESV_USAGE_READ;
492
493	num_fences = 0;
494	dma_fence_unwrap_for_each(f, &iter, fence)
495	++num_fences;
496
497	if (num_fences > 0) {
498	dma_resv_lock(obj: dmabuf->resv, NULL);
499
500	ret = dma_resv_reserve_fences(obj: dmabuf->resv, num_fences);
501	if (!ret) {
502	dma_fence_unwrap_for_each(f, &iter, fence)
503	dma_resv_add_fence(obj: dmabuf->resv, fence: f, usage);
504	}
505
506	dma_resv_unlock(obj: dmabuf->resv);
507	}
508
509	dma_fence_put(fence);
510
511	return ret;
512	}
513	#endif
514
515	static long dma_buf_ioctl(struct file *file,
516	unsigned int cmd, unsigned long arg)
517	{
518	struct dma_buf *dmabuf;
519	struct dma_buf_sync sync;
520	enum dma_data_direction direction;
521	int ret;
522
523	dmabuf = file->private_data;
524
525	switch (cmd) {
526	case DMA_BUF_IOCTL_SYNC:
527	if (copy_from_user(to: &sync, from: (void __user *) arg, n: sizeof(sync)))
528	return -EFAULT;
529
530	if (sync.flags & ~DMA_BUF_SYNC_VALID_FLAGS_MASK)
531	return -EINVAL;
532
533	switch (sync.flags & DMA_BUF_SYNC_RW) {
534	case DMA_BUF_SYNC_READ:
535	direction = DMA_FROM_DEVICE;
536	break;
537	case DMA_BUF_SYNC_WRITE:
538	direction = DMA_TO_DEVICE;
539	break;
540	case DMA_BUF_SYNC_RW:
541	direction = DMA_BIDIRECTIONAL;
542	break;
543	default:
544	return -EINVAL;
545	}
546
547	if (sync.flags & DMA_BUF_SYNC_END)
548	ret = dma_buf_end_cpu_access(dma_buf: dmabuf, dir: direction);
549	else
550	ret = dma_buf_begin_cpu_access(dma_buf: dmabuf, dir: direction);
551
552	return ret;
553
554	case DMA_BUF_SET_NAME_A:
555	case DMA_BUF_SET_NAME_B:
556	return dma_buf_set_name(dmabuf, buf: (const char __user *)arg);
557
558	#if IS_ENABLED(CONFIG_SYNC_FILE)
559	case DMA_BUF_IOCTL_EXPORT_SYNC_FILE:
560	return dma_buf_export_sync_file(dmabuf, user_data: (void __user *)arg);
561	case DMA_BUF_IOCTL_IMPORT_SYNC_FILE:
562	return dma_buf_import_sync_file(dmabuf, user_data: (const void __user *)arg);
563	#endif
564
565	default:
566	return -ENOTTY;
567	}
568	}
569
570	static void dma_buf_show_fdinfo(struct seq_file *m, struct file *file)
571	{
572	struct dma_buf *dmabuf = file->private_data;
573
574	seq_printf(m, fmt: "size:\t%zu\n", dmabuf->size);
575	/* Don't count the temporary reference taken inside procfs seq_show */
576	seq_printf(m, fmt: "count:\t%ld\n", file_count(dmabuf->file) - 1);
577	seq_printf(m, fmt: "exp_name:\t%s\n", dmabuf->exp_name);
578	spin_lock(lock: &dmabuf->name_lock);
579	if (dmabuf->name)
580	seq_printf(m, fmt: "name:\t%s\n", dmabuf->name);
581	spin_unlock(lock: &dmabuf->name_lock);
582	}
583
584	static const struct file_operations dma_buf_fops = {
585	.release = dma_buf_file_release,
586	.mmap = dma_buf_mmap_internal,
587	.llseek = dma_buf_llseek,
588	.poll = dma_buf_poll,
589	.unlocked_ioctl = dma_buf_ioctl,
590	.compat_ioctl = compat_ptr_ioctl,
591	.show_fdinfo = dma_buf_show_fdinfo,
592	};
593
594	/*
595	* is_dma_buf_file - Check if struct file* is associated with dma_buf
596	*/
597	static inline int is_dma_buf_file(struct file *file)
598	{
599	return file->f_op == &dma_buf_fops;
600	}
601
602	static struct file *dma_buf_getfile(size_t size, int flags)
603	{
604	static atomic64_t dmabuf_inode = ATOMIC64_INIT(0);
605	struct inode *inode = alloc_anon_inode(dma_buf_mnt->mnt_sb);
606	struct file *file;
607
608	if (IS_ERR(ptr: inode))
609	return ERR_CAST(ptr: inode);
610
611	inode->i_size = size;
612	inode_set_bytes(inode, bytes: size);
613
614	/*
615	* The ->i_ino acquired from get_next_ino() is not unique thus
616	* not suitable for using it as dentry name by dmabuf stats.
617	* Override ->i_ino with the unique and dmabuffs specific
618	* value.
619	*/
620	inode->i_ino = atomic64_inc_return(v: &dmabuf_inode);
621	flags &= O_ACCMODE | O_NONBLOCK;
622	file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf",
623	flags, &dma_buf_fops);
624	if (IS_ERR(ptr: file))
625	goto err_alloc_file;
626
627	return file;
628
629	err_alloc_file:
630	iput(inode);
631	return file;
632	}
633
634	/**
635	* DOC: dma buf device access
636	*
637	* For device DMA access to a shared DMA buffer the usual sequence of operations
638	* is fairly simple:
639	*
640	* 1. The exporter defines his exporter instance using
641	* DEFINE_DMA_BUF_EXPORT_INFO() and calls dma_buf_export() to wrap a private
642	* buffer object into a &dma_buf. It then exports that &dma_buf to userspace
643	* as a file descriptor by calling dma_buf_fd().
644	*
645	* 2. Userspace passes this file-descriptors to all drivers it wants this buffer
646	* to share with: First the file descriptor is converted to a &dma_buf using
647	* dma_buf_get(). Then the buffer is attached to the device using
648	* dma_buf_attach().
649	*
650	* Up to this stage the exporter is still free to migrate or reallocate the
651	* backing storage.
652	*
653	* 3. Once the buffer is attached to all devices userspace can initiate DMA
654	* access to the shared buffer. In the kernel this is done by calling
655	* dma_buf_map_attachment() and dma_buf_unmap_attachment().
656	*
657	* 4. Once a driver is done with a shared buffer it needs to call
658	* dma_buf_detach() (after cleaning up any mappings) and then release the
659	* reference acquired with dma_buf_get() by calling dma_buf_put().
660	*
661	* For the detailed semantics exporters are expected to implement see
662	* &dma_buf_ops.
663	*/
664
665	/**
666	* dma_buf_export - Creates a new dma_buf, and associates an anon file
667	* with this buffer, so it can be exported.
668	* Also connect the allocator specific data and ops to the buffer.
669	* Additionally, provide a name string for exporter; useful in debugging.
670	*
671	* @exp_info: [in] holds all the export related information provided
672	* by the exporter. see &struct dma_buf_export_info
673	* for further details.
674	*
675	* Returns, on success, a newly created struct dma_buf object, which wraps the
676	* supplied private data and operations for struct dma_buf_ops. On either
677	* missing ops, or error in allocating struct dma_buf, will return negative
678	* error.
679	*
680	* For most cases the easiest way to create @exp_info is through the
681	* %DEFINE_DMA_BUF_EXPORT_INFO macro.
682	*/
683	struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
684	{
685	struct dma_buf *dmabuf;
686	struct dma_resv *resv = exp_info->resv;
687	struct file *file;
688	size_t alloc_size = sizeof(struct dma_buf);
689	int ret;
690
691	if (WARN_ON(!exp_info->priv || !exp_info->ops
692	|| !exp_info->ops->map_dma_buf
693	|| !exp_info->ops->unmap_dma_buf
694	|| !exp_info->ops->release))
695	return ERR_PTR(error: -EINVAL);
696
697	if (WARN_ON(!exp_info->ops->pin != !exp_info->ops->unpin))
698	return ERR_PTR(error: -EINVAL);
699
700	if (!try_module_get(module: exp_info->owner))
701	return ERR_PTR(error: -ENOENT);
702
703	file = dma_buf_getfile(size: exp_info->size, flags: exp_info->flags);
704	if (IS_ERR(ptr: file)) {
705	ret = PTR_ERR(ptr: file);
706	goto err_module;
707	}
708
709	if (!exp_info->resv)
710	alloc_size += sizeof(struct dma_resv);
711	else
712	/* prevent &dma_buf[1] == dma_buf->resv */
713	alloc_size += 1;
714	dmabuf = kzalloc(alloc_size, GFP_KERNEL);
715	if (!dmabuf) {
716	ret = -ENOMEM;
717	goto err_file;
718	}
719
720	dmabuf->priv = exp_info->priv;
721	dmabuf->ops = exp_info->ops;
722	dmabuf->size = exp_info->size;
723	dmabuf->exp_name = exp_info->exp_name;
724	dmabuf->owner = exp_info->owner;
725	spin_lock_init(&dmabuf->name_lock);
726	init_waitqueue_head(&dmabuf->poll);
727	dmabuf->cb_in.poll = dmabuf->cb_out.poll = &dmabuf->poll;
728	dmabuf->cb_in.active = dmabuf->cb_out.active = 0;
729	INIT_LIST_HEAD(list: &dmabuf->attachments);
730
731	if (!resv) {
732	dmabuf->resv = (struct dma_resv *)&dmabuf[1];
733	dma_resv_init(obj: dmabuf->resv);
734	} else {
735	dmabuf->resv = resv;
736	}
737
738	ret = dma_buf_stats_setup(dmabuf, file);
739	if (ret)
740	goto err_dmabuf;
741
742	file->private_data = dmabuf;
743	file->f_path.dentry->d_fsdata = dmabuf;
744	dmabuf->file = file;
745
746	__dma_buf_list_add(dmabuf);
747
748	return dmabuf;
749
750	err_dmabuf:
751	if (!resv)
752	dma_resv_fini(obj: dmabuf->resv);
753	kfree(objp: dmabuf);
754	err_file:
755	fput(file);
756	err_module:
757	module_put(module: exp_info->owner);
758	return ERR_PTR(error: ret);
759	}
760	EXPORT_SYMBOL_NS_GPL(dma_buf_export, "DMA_BUF");
761
762	/**
763	* dma_buf_fd - returns a file descriptor for the given struct dma_buf
764	* @dmabuf: [in] pointer to dma_buf for which fd is required.
765	* @flags: [in] flags to give to fd
766	*
767	* On success, returns an associated 'fd'. Else, returns error.
768	*/
769	int dma_buf_fd(struct dma_buf *dmabuf, int flags)
770	{
771	if (!dmabuf || !dmabuf->file)
772	return -EINVAL;
773
774	return FD_ADD(flags, dmabuf->file);
775	}
776	EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF");
777
778	/**
779	* dma_buf_get - returns the struct dma_buf related to an fd
780	* @fd: [in] fd associated with the struct dma_buf to be returned
781	*
782	* On success, returns the struct dma_buf associated with an fd; uses
783	* file's refcounting done by fget to increase refcount. returns ERR_PTR
784	* otherwise.
785	*/
786	struct dma_buf *dma_buf_get(int fd)
787	{
788	struct file *file;
789
790	file = fget(fd);
791
792	if (!file)
793	return ERR_PTR(error: -EBADF);
794
795	if (!is_dma_buf_file(file)) {
796	fput(file);
797	return ERR_PTR(error: -EINVAL);
798	}
799
800	return file->private_data;
801	}
802	EXPORT_SYMBOL_NS_GPL(dma_buf_get, "DMA_BUF");
803
804	/**
805	* dma_buf_put - decreases refcount of the buffer
806	* @dmabuf: [in] buffer to reduce refcount of
807	*
808	* Uses file's refcounting done implicitly by fput().
809	*
810	* If, as a result of this call, the refcount becomes 0, the 'release' file
811	* operation related to this fd is called. It calls &dma_buf_ops.release vfunc
812	* in turn, and frees the memory allocated for dmabuf when exported.
813	*/
814	void dma_buf_put(struct dma_buf *dmabuf)
815	{
816	if (WARN_ON(!dmabuf || !dmabuf->file))
817	return;
818
819	fput(dmabuf->file);
820	}
821	EXPORT_SYMBOL_NS_GPL(dma_buf_put, "DMA_BUF");
822
823	static void mangle_sg_table(struct sg_table *sg_table)
824	{
825	#ifdef CONFIG_DMABUF_DEBUG
826	int i;
827	struct scatterlist *sg;
828
829	/* To catch abuse of the underlying struct page by importers mix
830	* up the bits, but take care to preserve the low SG_ bits to
831	* not corrupt the sgt. The mixing is undone on unmap
832	* before passing the sgt back to the exporter.
833	*/
834	for_each_sgtable_sg(sg_table, sg, i)
835	sg->page_link ^= ~0xffUL;
836	#endif
837
838	}
839
840	static inline bool
841	dma_buf_attachment_is_dynamic(struct dma_buf_attachment *attach)
842	{
843	return !!attach->importer_ops;
844	}
845
846	static bool
847	dma_buf_pin_on_map(struct dma_buf_attachment *attach)
848	{
849	return attach->dmabuf->ops->pin &&
850	(!dma_buf_attachment_is_dynamic(attach) ||
851	!IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY));
852	}
853
854	/**
855	* DOC: locking convention
856	*
857	* In order to avoid deadlock situations between dma-buf exports and importers,
858	* all dma-buf API users must follow the common dma-buf locking convention.
859	*
860	* Convention for importers
861	*
862	* 1. Importers must hold the dma-buf reservation lock when calling these
863	* functions:
864	*
865	* - dma_buf_pin()
866	* - dma_buf_unpin()
867	* - dma_buf_map_attachment()
868	* - dma_buf_unmap_attachment()
869	* - dma_buf_vmap()
870	* - dma_buf_vunmap()
871	*
872	* 2. Importers must not hold the dma-buf reservation lock when calling these
873	* functions:
874	*
875	* - dma_buf_attach()
876	* - dma_buf_dynamic_attach()
877	* - dma_buf_detach()
878	* - dma_buf_export()
879	* - dma_buf_fd()
880	* - dma_buf_get()
881	* - dma_buf_put()
882	* - dma_buf_mmap()
883	* - dma_buf_begin_cpu_access()
884	* - dma_buf_end_cpu_access()
885	* - dma_buf_map_attachment_unlocked()
886	* - dma_buf_unmap_attachment_unlocked()
887	* - dma_buf_vmap_unlocked()
888	* - dma_buf_vunmap_unlocked()
889	*
890	* Convention for exporters
891	*
892	* 1. These &dma_buf_ops callbacks are invoked with unlocked dma-buf
893	* reservation and exporter can take the lock:
894	*
895	* - &dma_buf_ops.attach()
896	* - &dma_buf_ops.detach()
897	* - &dma_buf_ops.release()
898	* - &dma_buf_ops.begin_cpu_access()
899	* - &dma_buf_ops.end_cpu_access()
900	* - &dma_buf_ops.mmap()
901	*
902	* 2. These &dma_buf_ops callbacks are invoked with locked dma-buf
903	* reservation and exporter can't take the lock:
904	*
905	* - &dma_buf_ops.pin()
906	* - &dma_buf_ops.unpin()
907	* - &dma_buf_ops.map_dma_buf()
908	* - &dma_buf_ops.unmap_dma_buf()
909	* - &dma_buf_ops.vmap()
910	* - &dma_buf_ops.vunmap()
911	*
912	* 3. Exporters must hold the dma-buf reservation lock when calling these
913	* functions:
914	*
915	* - dma_buf_move_notify()
916	*/
917
918	/**
919	* dma_buf_dynamic_attach - Add the device to dma_buf's attachments list
920	* @dmabuf: [in] buffer to attach device to.
921	* @dev: [in] device to be attached.
922	* @importer_ops: [in] importer operations for the attachment
923	* @importer_priv: [in] importer private pointer for the attachment
924	*
925	* Returns struct dma_buf_attachment pointer for this attachment. Attachments
926	* must be cleaned up by calling dma_buf_detach().
927	*
928	* Optionally this calls &dma_buf_ops.attach to allow device-specific attach
929	* functionality.
930	*
931	* Returns:
932	*
933	* A pointer to newly created &dma_buf_attachment on success, or a negative
934	* error code wrapped into a pointer on failure.
935	*
936	* Note that this can fail if the backing storage of @dmabuf is in a place not
937	* accessible to @dev, and cannot be moved to a more suitable place. This is
938	* indicated with the error code -EBUSY.
939	*/
940	struct dma_buf_attachment *
941	dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
942	const struct dma_buf_attach_ops *importer_ops,
943	void *importer_priv)
944	{
945	struct dma_buf_attachment *attach;
946	int ret;
947
948	if (WARN_ON(!dmabuf || !dev))
949	return ERR_PTR(error: -EINVAL);
950
951	if (WARN_ON(importer_ops && !importer_ops->move_notify))
952	return ERR_PTR(error: -EINVAL);
953
954	attach = kzalloc(sizeof(*attach), GFP_KERNEL);
955	if (!attach)
956	return ERR_PTR(error: -ENOMEM);
957
958	attach->dev = dev;
959	attach->dmabuf = dmabuf;
960	if (importer_ops)
961	attach->peer2peer = importer_ops->allow_peer2peer;
962	attach->importer_ops = importer_ops;
963	attach->importer_priv = importer_priv;
964
965	if (dmabuf->ops->attach) {
966	ret = dmabuf->ops->attach(dmabuf, attach);
967	if (ret)
968	goto err_attach;
969	}
970	dma_resv_lock(obj: dmabuf->resv, NULL);
971	list_add(new: &attach->node, head: &dmabuf->attachments);
972	dma_resv_unlock(obj: dmabuf->resv);
973
974	return attach;
975
976	err_attach:
977	kfree(objp: attach);
978	return ERR_PTR(error: ret);
979	}
980	EXPORT_SYMBOL_NS_GPL(dma_buf_dynamic_attach, "DMA_BUF");
981
982	/**
983	* dma_buf_attach - Wrapper for dma_buf_dynamic_attach
984	* @dmabuf: [in] buffer to attach device to.
985	* @dev: [in] device to be attached.
986	*
987	* Wrapper to call dma_buf_dynamic_attach() for drivers which still use a static
988	* mapping.
989	*/
990	struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf,
991	struct device *dev)
992	{
993	return dma_buf_dynamic_attach(dmabuf, dev, NULL, NULL);
994	}
995	EXPORT_SYMBOL_NS_GPL(dma_buf_attach, "DMA_BUF");
996
997	/**
998	* dma_buf_detach - Remove the given attachment from dmabuf's attachments list
999	* @dmabuf: [in] buffer to detach from.
1000	* @attach: [in] attachment to be detached; is free'd after this call.
1001	*
1002	* Clean up a device attachment obtained by calling dma_buf_attach().
1003	*
1004	* Optionally this calls &dma_buf_ops.detach for device-specific detach.
1005	*/
1006	void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)
1007	{
1008	if (WARN_ON(!dmabuf || !attach || dmabuf != attach->dmabuf))
1009	return;
1010
1011	dma_resv_lock(obj: dmabuf->resv, NULL);
1012	list_del(entry: &attach->node);
1013	dma_resv_unlock(obj: dmabuf->resv);
1014
1015	if (dmabuf->ops->detach)
1016	dmabuf->ops->detach(dmabuf, attach);
1017
1018	kfree(objp: attach);
1019	}
1020	EXPORT_SYMBOL_NS_GPL(dma_buf_detach, "DMA_BUF");
1021
1022	/**
1023	* dma_buf_pin - Lock down the DMA-buf
1024	* @attach: [in] attachment which should be pinned
1025	*
1026	* Only dynamic importers (who set up @attach with dma_buf_dynamic_attach()) may
1027	* call this, and only for limited use cases like scanout and not for temporary
1028	* pin operations. It is not permitted to allow userspace to pin arbitrary
1029	* amounts of buffers through this interface.
1030	*
1031	* Buffers must be unpinned by calling dma_buf_unpin().
1032	*
1033	* Returns:
1034	* 0 on success, negative error code on failure.
1035	*/
1036	int dma_buf_pin(struct dma_buf_attachment *attach)
1037	{
1038	struct dma_buf *dmabuf = attach->dmabuf;
1039	int ret = 0;
1040
1041	WARN_ON(!attach->importer_ops);
1042
1043	dma_resv_assert_held(dmabuf->resv);
1044
1045	if (dmabuf->ops->pin)
1046	ret = dmabuf->ops->pin(attach);
1047
1048	return ret;
1049	}
1050	EXPORT_SYMBOL_NS_GPL(dma_buf_pin, "DMA_BUF");
1051
1052	/**
1053	* dma_buf_unpin - Unpin a DMA-buf
1054	* @attach: [in] attachment which should be unpinned
1055	*
1056	* This unpins a buffer pinned by dma_buf_pin() and allows the exporter to move
1057	* any mapping of @attach again and inform the importer through
1058	* &dma_buf_attach_ops.move_notify.
1059	*/
1060	void dma_buf_unpin(struct dma_buf_attachment *attach)
1061	{
1062	struct dma_buf *dmabuf = attach->dmabuf;
1063
1064	WARN_ON(!attach->importer_ops);
1065
1066	dma_resv_assert_held(dmabuf->resv);
1067
1068	if (dmabuf->ops->unpin)
1069	dmabuf->ops->unpin(attach);
1070	}
1071	EXPORT_SYMBOL_NS_GPL(dma_buf_unpin, "DMA_BUF");
1072
1073	/**
1074	* dma_buf_map_attachment - Returns the scatterlist table of the attachment;
1075	* mapped into _device_ address space. Is a wrapper for map_dma_buf() of the
1076	* dma_buf_ops.
1077	* @attach: [in] attachment whose scatterlist is to be returned
1078	* @direction: [in] direction of DMA transfer
1079	*
1080	* Returns sg_table containing the scatterlist to be returned; returns ERR_PTR
1081	* on error. May return -EINTR if it is interrupted by a signal.
1082	*
1083	* On success, the DMA addresses and lengths in the returned scatterlist are
1084	* PAGE_SIZE aligned.
1085	*
1086	* A mapping must be unmapped by using dma_buf_unmap_attachment(). Note that
1087	* the underlying backing storage is pinned for as long as a mapping exists,
1088	* therefore users/importers should not hold onto a mapping for undue amounts of
1089	* time.
1090	*
1091	* Important: Dynamic importers must wait for the exclusive fence of the struct
1092	* dma_resv attached to the DMA-BUF first.
1093	*/
1094	struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
1095	enum dma_data_direction direction)
1096	{
1097	struct sg_table *sg_table;
1098	signed long ret;
1099
1100	might_sleep();
1101
1102	if (WARN_ON(!attach || !attach->dmabuf))
1103	return ERR_PTR(error: -EINVAL);
1104
1105	dma_resv_assert_held(attach->dmabuf->resv);
1106
1107	if (dma_buf_pin_on_map(attach)) {
1108	ret = attach->dmabuf->ops->pin(attach);
1109	/*
1110	* Catch exporters making buffers inaccessible even when
1111	* attachments preventing that exist.
1112	*/
1113	WARN_ON_ONCE(ret == -EBUSY);
1114	if (ret)
1115	return ERR_PTR(error: ret);
1116	}
1117
1118	sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction);
1119	if (!sg_table)
1120	sg_table = ERR_PTR(error: -ENOMEM);
1121	if (IS_ERR(ptr: sg_table))
1122	goto error_unpin;
1123
1124	/*
1125	* Importers with static attachments don't wait for fences.
1126	*/
1127	if (!dma_buf_attachment_is_dynamic(attach)) {
1128	ret = dma_resv_wait_timeout(obj: attach->dmabuf->resv,
1129	usage: DMA_RESV_USAGE_KERNEL, intr: true,
1130	MAX_SCHEDULE_TIMEOUT);
1131	if (ret < 0)
1132	goto error_unmap;
1133	}
1134	mangle_sg_table(sg_table);
1135
1136	#ifdef CONFIG_DMA_API_DEBUG
1137	{
1138	struct scatterlist *sg;
1139	u64 addr;
1140	int len;
1141	int i;
1142
1143	for_each_sgtable_dma_sg(sg_table, sg, i) {
1144	addr = sg_dma_address(sg);
1145	len = sg_dma_len(sg);
1146	if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(len)) {
1147	pr_debug("%s: addr %llx or len %x is not page aligned!\n",
1148	__func__, addr, len);
1149	}
1150	}
1151	}
1152	#endif /* CONFIG_DMA_API_DEBUG */
1153	return sg_table;
1154
1155	error_unmap:
1156	attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);
1157	sg_table = ERR_PTR(error: ret);
1158
1159	error_unpin:
1160	if (dma_buf_pin_on_map(attach))
1161	attach->dmabuf->ops->unpin(attach);
1162
1163	return sg_table;
1164	}
1165	EXPORT_SYMBOL_NS_GPL(dma_buf_map_attachment, "DMA_BUF");
1166
1167	/**
1168	* dma_buf_map_attachment_unlocked - Returns the scatterlist table of the attachment;
1169	* mapped into _device_ address space. Is a wrapper for map_dma_buf() of the
1170	* dma_buf_ops.
1171	* @attach: [in] attachment whose scatterlist is to be returned
1172	* @direction: [in] direction of DMA transfer
1173	*
1174	* Unlocked variant of dma_buf_map_attachment().
1175	*/
1176	struct sg_table *
1177	dma_buf_map_attachment_unlocked(struct dma_buf_attachment *attach,
1178	enum dma_data_direction direction)
1179	{
1180	struct sg_table *sg_table;
1181
1182	might_sleep();
1183
1184	if (WARN_ON(!attach || !attach->dmabuf))
1185	return ERR_PTR(error: -EINVAL);
1186
1187	dma_resv_lock(obj: attach->dmabuf->resv, NULL);
1188	sg_table = dma_buf_map_attachment(attach, direction);
1189	dma_resv_unlock(obj: attach->dmabuf->resv);
1190
1191	return sg_table;
1192	}
1193	EXPORT_SYMBOL_NS_GPL(dma_buf_map_attachment_unlocked, "DMA_BUF");
1194
1195	/**
1196	* dma_buf_unmap_attachment - unmaps and decreases usecount of the buffer;might
1197	* deallocate the scatterlist associated. Is a wrapper for unmap_dma_buf() of
1198	* dma_buf_ops.
1199	* @attach: [in] attachment to unmap buffer from
1200	* @sg_table: [in] scatterlist info of the buffer to unmap
1201	* @direction: [in] direction of DMA transfer
1202	*
1203	* This unmaps a DMA mapping for @attached obtained by dma_buf_map_attachment().
1204	*/
1205	void dma_buf_unmap_attachment(struct dma_buf_attachment *attach,
1206	struct sg_table *sg_table,
1207	enum dma_data_direction direction)
1208	{
1209	might_sleep();
1210
1211	if (WARN_ON(!attach || !attach->dmabuf || !sg_table))
1212	return;
1213
1214	dma_resv_assert_held(attach->dmabuf->resv);
1215
1216	mangle_sg_table(sg_table);
1217	attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);
1218
1219	if (dma_buf_pin_on_map(attach))
1220	attach->dmabuf->ops->unpin(attach);
1221	}
1222	EXPORT_SYMBOL_NS_GPL(dma_buf_unmap_attachment, "DMA_BUF");
1223
1224	/**
1225	* dma_buf_unmap_attachment_unlocked - unmaps and decreases usecount of the buffer;might
1226	* deallocate the scatterlist associated. Is a wrapper for unmap_dma_buf() of
1227	* dma_buf_ops.
1228	* @attach: [in] attachment to unmap buffer from
1229	* @sg_table: [in] scatterlist info of the buffer to unmap
1230	* @direction: [in] direction of DMA transfer
1231	*
1232	* Unlocked variant of dma_buf_unmap_attachment().
1233	*/
1234	void dma_buf_unmap_attachment_unlocked(struct dma_buf_attachment *attach,
1235	struct sg_table *sg_table,
1236	enum dma_data_direction direction)
1237	{
1238	might_sleep();
1239
1240	if (WARN_ON(!attach || !attach->dmabuf || !sg_table))
1241	return;
1242
1243	dma_resv_lock(obj: attach->dmabuf->resv, NULL);
1244	dma_buf_unmap_attachment(attach, sg_table, direction);
1245	dma_resv_unlock(obj: attach->dmabuf->resv);
1246	}
1247	EXPORT_SYMBOL_NS_GPL(dma_buf_unmap_attachment_unlocked, "DMA_BUF");
1248
1249	/**
1250	* dma_buf_move_notify - notify attachments that DMA-buf is moving
1251	*
1252	* @dmabuf: [in] buffer which is moving
1253	*
1254	* Informs all attachments that they need to destroy and recreate all their
1255	* mappings.
1256	*/
1257	void dma_buf_move_notify(struct dma_buf *dmabuf)
1258	{
1259	struct dma_buf_attachment *attach;
1260
1261	dma_resv_assert_held(dmabuf->resv);
1262
1263	list_for_each_entry(attach, &dmabuf->attachments, node)
1264	if (attach->importer_ops)
1265	attach->importer_ops->move_notify(attach);
1266	}
1267	EXPORT_SYMBOL_NS_GPL(dma_buf_move_notify, "DMA_BUF");
1268
1269	/**
1270	* DOC: cpu access
1271	*
1272	* There are multiple reasons for supporting CPU access to a dma buffer object:
1273	*
1274	* - Fallback operations in the kernel, for example when a device is connected
1275	* over USB and the kernel needs to shuffle the data around first before
1276	* sending it away. Cache coherency is handled by bracketing any transactions
1277	* with calls to dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
1278	* access.
1279	*
1280	* Since for most kernel internal dma-buf accesses need the entire buffer, a
1281	* vmap interface is introduced. Note that on very old 32-bit architectures
1282	* vmalloc space might be limited and result in vmap calls failing.
1283	*
1284	* Interfaces:
1285	*
1286	* .. code-block:: c
1287	*
1288	* void *dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
1289	* void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
1290	*
1291	* The vmap call can fail if there is no vmap support in the exporter, or if
1292	* it runs out of vmalloc space. Note that the dma-buf layer keeps a reference
1293	* count for all vmap access and calls down into the exporter's vmap function
1294	* only when no vmapping exists, and only unmaps it once. Protection against
1295	* concurrent vmap/vunmap calls is provided by taking the &dma_buf.lock mutex.
1296	*
1297	* - For full compatibility on the importer side with existing userspace
1298	* interfaces, which might already support mmap'ing buffers. This is needed in
1299	* many processing pipelines (e.g. feeding a software rendered image into a
1300	* hardware pipeline, thumbnail creation, snapshots, ...). Also, Android's ION
1301	* framework already supported this and for DMA buffer file descriptors to
1302	* replace ION buffers mmap support was needed.
1303	*
1304	* There is no special interfaces, userspace simply calls mmap on the dma-buf
1305	* fd. But like for CPU access there's a need to bracket the actual access,
1306	* which is handled by the ioctl (DMA_BUF_IOCTL_SYNC). Note that
1307	* DMA_BUF_IOCTL_SYNC can fail with -EAGAIN or -EINTR, in which case it must
1308	* be restarted.
1309	*
1310	* Some systems might need some sort of cache coherency management e.g. when
1311	* CPU and GPU domains are being accessed through dma-buf at the same time.
1312	* To circumvent this problem there are begin/end coherency markers, that
1313	* forward directly to existing dma-buf device drivers vfunc hooks. Userspace
1314	* can make use of those markers through the DMA_BUF_IOCTL_SYNC ioctl. The
1315	* sequence would be used like following:
1316	*
1317	* - mmap dma-buf fd
1318	* - for each drawing/upload cycle in CPU 1. SYNC_START ioctl, 2. read/write
1319	* to mmap area 3. SYNC_END ioctl. This can be repeated as often as you
1320	* want (with the new data being consumed by say the GPU or the scanout
1321	* device)
1322	* - munmap once you don't need the buffer any more
1323	*
1324	* For correctness and optimal performance, it is always required to use
1325	* SYNC_START and SYNC_END before and after, respectively, when accessing the
1326	* mapped address. Userspace cannot rely on coherent access, even when there
1327	* are systems where it just works without calling these ioctls.
1328	*
1329	* - And as a CPU fallback in userspace processing pipelines.
1330	*
1331	* Similar to the motivation for kernel cpu access it is again important that
1332	* the userspace code of a given importing subsystem can use the same
1333	* interfaces with a imported dma-buf buffer object as with a native buffer
1334	* object. This is especially important for drm where the userspace part of
1335	* contemporary OpenGL, X, and other drivers is huge, and reworking them to
1336	* use a different way to mmap a buffer rather invasive.
1337	*
1338	* The assumption in the current dma-buf interfaces is that redirecting the
1339	* initial mmap is all that's needed. A survey of some of the existing
1340	* subsystems shows that no driver seems to do any nefarious thing like
1341	* syncing up with outstanding asynchronous processing on the device or
1342	* allocating special resources at fault time. So hopefully this is good
1343	* enough, since adding interfaces to intercept pagefaults and allow pte
1344	* shootdowns would increase the complexity quite a bit.
1345	*
1346	* Interface:
1347	*
1348	* .. code-block:: c
1349	*
1350	* int dma_buf_mmap(struct dma_buf *, struct vm_area_struct *, unsigned long);
1351	*
1352	* If the importing subsystem simply provides a special-purpose mmap call to
1353	* set up a mapping in userspace, calling do_mmap with &dma_buf.file will
1354	* equally achieve that for a dma-buf object.
1355	*/
1356
1357	static int __dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
1358	enum dma_data_direction direction)
1359	{
1360	bool write = (direction == DMA_BIDIRECTIONAL ||
1361	direction == DMA_TO_DEVICE);
1362	struct dma_resv *resv = dmabuf->resv;
1363	long ret;
1364
1365	/* Wait on any implicit rendering fences */
1366	ret = dma_resv_wait_timeout(obj: resv, usage: dma_resv_usage_rw(write),
1367	intr: true, MAX_SCHEDULE_TIMEOUT);
1368	if (ret < 0)
1369	return ret;
1370
1371	return 0;
1372	}
1373
1374	/**
1375	* dma_buf_begin_cpu_access - Must be called before accessing a dma_buf from the
1376	* cpu in the kernel context. Calls begin_cpu_access to allow exporter-specific
1377	* preparations. Coherency is only guaranteed in the specified range for the
1378	* specified access direction.
1379	* @dmabuf: [in] buffer to prepare cpu access for.
1380	* @direction: [in] direction of access.
1381	*
1382	* After the cpu access is complete the caller should call
1383	* dma_buf_end_cpu_access(). Only when cpu access is bracketed by both calls is
1384	* it guaranteed to be coherent with other DMA access.
1385	*
1386	* This function will also wait for any DMA transactions tracked through
1387	* implicit synchronization in &dma_buf.resv. For DMA transactions with explicit
1388	* synchronization this function will only ensure cache coherency, callers must
1389	* ensure synchronization with such DMA transactions on their own.
1390	*
1391	* Can return negative error values, returns 0 on success.
1392	*/
1393	int dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
1394	enum dma_data_direction direction)
1395	{
1396	int ret = 0;
1397
1398	if (WARN_ON(!dmabuf))
1399	return -EINVAL;
1400
1401	might_lock(&dmabuf->resv->lock.base);
1402
1403	if (dmabuf->ops->begin_cpu_access)
1404	ret = dmabuf->ops->begin_cpu_access(dmabuf, direction);
1405
1406	/* Ensure that all fences are waited upon - but we first allow
1407	* the native handler the chance to do so more efficiently if it
1408	* chooses. A double invocation here will be reasonably cheap no-op.
1409	*/
1410	if (ret == 0)
1411	ret = __dma_buf_begin_cpu_access(dmabuf, direction);
1412
1413	return ret;
1414	}
1415	EXPORT_SYMBOL_NS_GPL(dma_buf_begin_cpu_access, "DMA_BUF");
1416
1417	/**
1418	* dma_buf_end_cpu_access - Must be called after accessing a dma_buf from the
1419	* cpu in the kernel context. Calls end_cpu_access to allow exporter-specific
1420	* actions. Coherency is only guaranteed in the specified range for the
1421	* specified access direction.
1422	* @dmabuf: [in] buffer to complete cpu access for.
1423	* @direction: [in] direction of access.
1424	*
1425	* This terminates CPU access started with dma_buf_begin_cpu_access().
1426	*
1427	* Can return negative error values, returns 0 on success.
1428	*/
1429	int dma_buf_end_cpu_access(struct dma_buf *dmabuf,
1430	enum dma_data_direction direction)
1431	{
1432	int ret = 0;
1433
1434	WARN_ON(!dmabuf);
1435
1436	might_lock(&dmabuf->resv->lock.base);
1437
1438	if (dmabuf->ops->end_cpu_access)
1439	ret = dmabuf->ops->end_cpu_access(dmabuf, direction);
1440
1441	return ret;
1442	}
1443	EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, "DMA_BUF");
1444
1445
1446	/**
1447	* dma_buf_mmap - Setup up a userspace mmap with the given vma
1448	* @dmabuf: [in] buffer that should back the vma
1449	* @vma: [in] vma for the mmap
1450	* @pgoff: [in] offset in pages where this mmap should start within the
1451	* dma-buf buffer.
1452	*
1453	* This function adjusts the passed in vma so that it points at the file of the
1454	* dma_buf operation. It also adjusts the starting pgoff and does bounds
1455	* checking on the size of the vma. Then it calls the exporters mmap function to
1456	* set up the mapping.
1457	*
1458	* Can return negative error values, returns 0 on success.
1459	*/
1460	int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,
1461	unsigned long pgoff)
1462	{
1463	if (WARN_ON(!dmabuf || !vma))
1464	return -EINVAL;
1465
1466	/* check if buffer supports mmap */
1467	if (!dmabuf->ops->mmap)
1468	return -EINVAL;
1469
1470	/* check for offset overflow */
1471	if (pgoff + vma_pages(vma) < pgoff)
1472	return -EOVERFLOW;
1473
1474	/* check for overflowing the buffer's size */
1475	if (pgoff + vma_pages(vma) >
1476	dmabuf->size >> PAGE_SHIFT)
1477	return -EINVAL;
1478
1479	/* readjust the vma */
1480	vma_set_file(vma, file: dmabuf->file);
1481	vma->vm_pgoff = pgoff;
1482
1483	return dmabuf->ops->mmap(dmabuf, vma);
1484	}
1485	EXPORT_SYMBOL_NS_GPL(dma_buf_mmap, "DMA_BUF");
1486
1487	/**
1488	* dma_buf_vmap - Create virtual mapping for the buffer object into kernel
1489	* address space. Same restrictions as for vmap and friends apply.
1490	* @dmabuf: [in] buffer to vmap
1491	* @map: [out] returns the vmap pointer
1492	*
1493	* This call may fail due to lack of virtual mapping address space.
1494	* These calls are optional in drivers. The intended use for them
1495	* is for mapping objects linear in kernel space for high use objects.
1496	*
1497	* To ensure coherency users must call dma_buf_begin_cpu_access() and
1498	* dma_buf_end_cpu_access() around any cpu access performed through this
1499	* mapping.
1500	*
1501	* Returns 0 on success, or a negative errno code otherwise.
1502	*/
1503	int dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
1504	{
1505	struct iosys_map ptr;
1506	int ret;
1507
1508	iosys_map_clear(map);
1509
1510	if (WARN_ON(!dmabuf))
1511	return -EINVAL;
1512
1513	dma_resv_assert_held(dmabuf->resv);
1514
1515	if (!dmabuf->ops->vmap)
1516	return -EINVAL;
1517
1518	if (dmabuf->vmapping_counter) {
1519	dmabuf->vmapping_counter++;
1520	BUG_ON(iosys_map_is_null(&dmabuf->vmap_ptr));
1521	*map = dmabuf->vmap_ptr;
1522	return 0;
1523	}
1524
1525	BUG_ON(iosys_map_is_set(&dmabuf->vmap_ptr));
1526
1527	ret = dmabuf->ops->vmap(dmabuf, &ptr);
1528	if (WARN_ON_ONCE(ret))
1529	return ret;
1530
1531	dmabuf->vmap_ptr = ptr;
1532	dmabuf->vmapping_counter = 1;
1533
1534	*map = dmabuf->vmap_ptr;
1535
1536	return 0;
1537	}
1538	EXPORT_SYMBOL_NS_GPL(dma_buf_vmap, "DMA_BUF");
1539
1540	/**
1541	* dma_buf_vmap_unlocked - Create virtual mapping for the buffer object into kernel
1542	* address space. Same restrictions as for vmap and friends apply.
1543	* @dmabuf: [in] buffer to vmap
1544	* @map: [out] returns the vmap pointer
1545	*
1546	* Unlocked version of dma_buf_vmap()
1547	*
1548	* Returns 0 on success, or a negative errno code otherwise.
1549	*/
1550	int dma_buf_vmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map)
1551	{
1552	int ret;
1553
1554	iosys_map_clear(map);
1555
1556	if (WARN_ON(!dmabuf))
1557	return -EINVAL;
1558
1559	dma_resv_lock(obj: dmabuf->resv, NULL);
1560	ret = dma_buf_vmap(dmabuf, map);
1561	dma_resv_unlock(obj: dmabuf->resv);
1562
1563	return ret;
1564	}
1565	EXPORT_SYMBOL_NS_GPL(dma_buf_vmap_unlocked, "DMA_BUF");
1566
1567	/**
1568	* dma_buf_vunmap - Unmap a vmap obtained by dma_buf_vmap.
1569	* @dmabuf: [in] buffer to vunmap
1570	* @map: [in] vmap pointer to vunmap
1571	*/
1572	void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
1573	{
1574	if (WARN_ON(!dmabuf))
1575	return;
1576
1577	dma_resv_assert_held(dmabuf->resv);
1578
1579	BUG_ON(iosys_map_is_null(&dmabuf->vmap_ptr));
1580	BUG_ON(dmabuf->vmapping_counter == 0);
1581	BUG_ON(!iosys_map_is_equal(&dmabuf->vmap_ptr, map));
1582
1583	if (--dmabuf->vmapping_counter == 0) {
1584	if (dmabuf->ops->vunmap)
1585	dmabuf->ops->vunmap(dmabuf, map);
1586	iosys_map_clear(map: &dmabuf->vmap_ptr);
1587	}
1588	}
1589	EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap, "DMA_BUF");
1590
1591	/**
1592	* dma_buf_vunmap_unlocked - Unmap a vmap obtained by dma_buf_vmap.
1593	* @dmabuf: [in] buffer to vunmap
1594	* @map: [in] vmap pointer to vunmap
1595	*/
1596	void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map)
1597	{
1598	if (WARN_ON(!dmabuf))
1599	return;
1600
1601	dma_resv_lock(obj: dmabuf->resv, NULL);
1602	dma_buf_vunmap(dmabuf, map);
1603	dma_resv_unlock(obj: dmabuf->resv);
1604	}
1605	EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap_unlocked, "DMA_BUF");
1606
1607	#ifdef CONFIG_DEBUG_FS
1608	static int dma_buf_debug_show(struct seq_file *s, void *unused)
1609	{
1610	struct dma_buf *buf_obj;
1611	struct dma_buf_attachment *attach_obj;
1612	int count = 0, attach_count;
1613	size_t size = 0;
1614	int ret;
1615
1616	ret = mutex_lock_interruptible(&dmabuf_list_mutex);
1617
1618	if (ret)
1619	return ret;
1620
1621	seq_puts(m: s, s: "\nDma-buf Objects:\n");
1622	seq_printf(m: s, fmt: "%-8s\t%-8s\t%-8s\t%-8s\texp_name\t%-8s\tname\n",
1623	"size", "flags", "mode", "count", "ino");
1624
1625	list_for_each_entry(buf_obj, &dmabuf_list, list_node) {
1626
1627	ret = dma_resv_lock_interruptible(obj: buf_obj->resv, NULL);
1628	if (ret)
1629	goto error_unlock;
1630
1631
1632	spin_lock(lock: &buf_obj->name_lock);
1633	seq_printf(m: s, fmt: "%08zu\t%08x\t%08x\t%08ld\t%s\t%08lu\t%s\n",
1634	buf_obj->size,
1635	buf_obj->file->f_flags, buf_obj->file->f_mode,
1636	file_count(buf_obj->file),
1637	buf_obj->exp_name,
1638	file_inode(f: buf_obj->file)->i_ino,
1639	buf_obj->name ?: "<none>");
1640	spin_unlock(lock: &buf_obj->name_lock);
1641
1642	dma_resv_describe(obj: buf_obj->resv, seq: s);
1643
1644	seq_puts(m: s, s: "\tAttached Devices:\n");
1645	attach_count = 0;
1646
1647	list_for_each_entry(attach_obj, &buf_obj->attachments, node) {
1648	seq_printf(m: s, fmt: "\t%s\n", dev_name(dev: attach_obj->dev));
1649	attach_count++;
1650	}
1651	dma_resv_unlock(obj: buf_obj->resv);
1652
1653	seq_printf(m: s, fmt: "Total %d devices attached\n\n",
1654	attach_count);
1655
1656	count++;
1657	size += buf_obj->size;
1658	}
1659
1660	seq_printf(m: s, fmt: "\nTotal %d objects, %zu bytes\n", count, size);
1661
1662	mutex_unlock(lock: &dmabuf_list_mutex);
1663	return 0;
1664
1665	error_unlock:
1666	mutex_unlock(lock: &dmabuf_list_mutex);
1667	return ret;
1668	}
1669
1670	DEFINE_SHOW_ATTRIBUTE(dma_buf_debug);
1671
1672	static struct dentry *dma_buf_debugfs_dir;
1673
1674	static int dma_buf_init_debugfs(void)
1675	{
1676	struct dentry *d;
1677	int err = 0;
1678
1679	d = debugfs_create_dir(name: "dma_buf", NULL);
1680	if (IS_ERR(ptr: d))
1681	return PTR_ERR(ptr: d);
1682
1683	dma_buf_debugfs_dir = d;
1684
1685	d = debugfs_create_file("bufinfo", 0444, dma_buf_debugfs_dir,
1686	NULL, &dma_buf_debug_fops);
1687	if (IS_ERR(ptr: d)) {
1688	pr_debug("dma_buf: debugfs: failed to create node bufinfo\n");
1689	debugfs_remove_recursive(dentry: dma_buf_debugfs_dir);
1690	dma_buf_debugfs_dir = NULL;
1691	err = PTR_ERR(ptr: d);
1692	}
1693
1694	return err;
1695	}
1696
1697	static void dma_buf_uninit_debugfs(void)
1698	{
1699	debugfs_remove_recursive(dentry: dma_buf_debugfs_dir);
1700	}
1701	#else
1702	static inline int dma_buf_init_debugfs(void)
1703	{
1704	return 0;
1705	}
1706	static inline void dma_buf_uninit_debugfs(void)
1707	{
1708	}
1709	#endif
1710
1711	static int __init dma_buf_init(void)
1712	{
1713	int ret;
1714
1715	ret = dma_buf_init_sysfs_statistics();
1716	if (ret)
1717	return ret;
1718
1719	dma_buf_mnt = kern_mount(&dma_buf_fs_type);
1720	if (IS_ERR(ptr: dma_buf_mnt))
1721	return PTR_ERR(ptr: dma_buf_mnt);
1722
1723	dma_buf_init_debugfs();
1724	return 0;
1725	}
1726	subsys_initcall(dma_buf_init);
1727
1728	static void __exit dma_buf_deinit(void)
1729	{
1730	dma_buf_uninit_debugfs();
1731	kern_unmount(mnt: dma_buf_mnt);
1732	dma_buf_uninit_sysfs_statistics();
1733	}
1734	__exitcall(dma_buf_deinit);
1735