1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Userspace block device - block device which IO is handled from userspace
4	*
5	* Take full use of io_uring passthrough command for communicating with
6	* ublk userspace daemon(ublksrvd) for handling basic IO request.
7	*
8	* Copyright 2022 Ming Lei <ming.lei@redhat.com>
9	*
10	* (part of code stolen from loop.c)
11	*/
12	#include <linux/module.h>
13	#include <linux/moduleparam.h>
14	#include <linux/sched.h>
15	#include <linux/fs.h>
16	#include <linux/pagemap.h>
17	#include <linux/file.h>
18	#include <linux/stat.h>
19	#include <linux/errno.h>
20	#include <linux/major.h>
21	#include <linux/wait.h>
22	#include <linux/blkdev.h>
23	#include <linux/init.h>
24	#include <linux/swap.h>
25	#include <linux/slab.h>
26	#include <linux/compat.h>
27	#include <linux/mutex.h>
28	#include <linux/writeback.h>
29	#include <linux/completion.h>
30	#include <linux/highmem.h>
31	#include <linux/sysfs.h>
32	#include <linux/miscdevice.h>
33	#include <linux/falloc.h>
34	#include <linux/uio.h>
35	#include <linux/ioprio.h>
36	#include <linux/sched/mm.h>
37	#include <linux/uaccess.h>
38	#include <linux/cdev.h>
39	#include <linux/io_uring/cmd.h>
40	#include <linux/blk-mq.h>
41	#include <linux/delay.h>
42	#include <linux/mm.h>
43	#include <asm/page.h>
44	#include <linux/task_work.h>
45	#include <linux/namei.h>
46	#include <linux/kref.h>
47	#include <uapi/linux/ublk_cmd.h>
48
49	#define UBLK_MINORS (1U << MINORBITS)
50
51	#define UBLK_INVALID_BUF_IDX ((u16)-1)
52
53	/* private ioctl command mirror */
54	#define UBLK_CMD_DEL_DEV_ASYNC _IOC_NR(UBLK_U_CMD_DEL_DEV_ASYNC)
55	#define UBLK_CMD_UPDATE_SIZE _IOC_NR(UBLK_U_CMD_UPDATE_SIZE)
56	#define UBLK_CMD_QUIESCE_DEV _IOC_NR(UBLK_U_CMD_QUIESCE_DEV)
57
58	#define UBLK_IO_REGISTER_IO_BUF _IOC_NR(UBLK_U_IO_REGISTER_IO_BUF)
59	#define UBLK_IO_UNREGISTER_IO_BUF _IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF)
60
61	/* All UBLK_F_* have to be included into UBLK_F_ALL */
62	#define UBLK_F_ALL (UBLK_F_SUPPORT_ZERO_COPY \
63	| UBLK_F_URING_CMD_COMP_IN_TASK \
64	| UBLK_F_NEED_GET_DATA \
65	| UBLK_F_USER_RECOVERY \
66	| UBLK_F_USER_RECOVERY_REISSUE \
67	| UBLK_F_UNPRIVILEGED_DEV \
68	| UBLK_F_CMD_IOCTL_ENCODE \
69	| UBLK_F_USER_COPY \
70	| UBLK_F_ZONED \
71	| UBLK_F_USER_RECOVERY_FAIL_IO \
72	| UBLK_F_UPDATE_SIZE \
73	| UBLK_F_AUTO_BUF_REG \
74	| UBLK_F_QUIESCE \
75	| UBLK_F_PER_IO_DAEMON \
76	| UBLK_F_BUF_REG_OFF_DAEMON)
77
78	#define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \
79	| UBLK_F_USER_RECOVERY_REISSUE \
80	| UBLK_F_USER_RECOVERY_FAIL_IO)
81
82	/* All UBLK_PARAM_TYPE_* should be included here */
83	#define UBLK_PARAM_TYPE_ALL \
84	(UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DISCARD | \
85	UBLK_PARAM_TYPE_DEVT | UBLK_PARAM_TYPE_ZONED | \
86	UBLK_PARAM_TYPE_DMA_ALIGN | UBLK_PARAM_TYPE_SEGMENT)
87
88	struct ublk_uring_cmd_pdu {
89	/*
90	* Store requests in same batch temporarily for queuing them to
91	* daemon context.
92	*
93	* It should have been stored to request payload, but we do want
94	* to avoid extra pre-allocation, and uring_cmd payload is always
95	* free for us
96	*/
97	union {
98	struct request *req;
99	struct request *req_list;
100	};
101
102	/*
103	* The following two are valid in this cmd whole lifetime, and
104	* setup in ublk uring_cmd handler
105	*/
106	struct ublk_queue *ubq;
107
108	u16 tag;
109	};
110
111	/*
112	* io command is active: sqe cmd is received, and its cqe isn't done
113	*
114	* If the flag is set, the io command is owned by ublk driver, and waited
115	* for incoming blk-mq request from the ublk block device.
116	*
117	* If the flag is cleared, the io command will be completed, and owned by
118	* ublk server.
119	*/
120	#define UBLK_IO_FLAG_ACTIVE 0x01
121
122	/*
123	* IO command is completed via cqe, and it is being handled by ublksrv, and
124	* not committed yet
125	*
126	* Basically exclusively with UBLK_IO_FLAG_ACTIVE, so can be served for
127	* cross verification
128	*/
129	#define UBLK_IO_FLAG_OWNED_BY_SRV 0x02
130
131	/*
132	* UBLK_IO_FLAG_NEED_GET_DATA is set because IO command requires
133	* get data buffer address from ublksrv.
134	*
135	* Then, bio data could be copied into this data buffer for a WRITE request
136	* after the IO command is issued again and UBLK_IO_FLAG_NEED_GET_DATA is unset.
137	*/
138	#define UBLK_IO_FLAG_NEED_GET_DATA 0x08
139
140	/*
141	* request buffer is registered automatically, so we have to unregister it
142	* before completing this request.
143	*
144	* io_uring will unregister buffer automatically for us during exiting.
145	*/
146	#define UBLK_IO_FLAG_AUTO_BUF_REG 0x10
147
148	/* atomic RW with ubq->cancel_lock */
149	#define UBLK_IO_FLAG_CANCELED 0x80000000
150
151	/*
152	* Initialize refcount to a large number to include any registered buffers.
153	* UBLK_IO_COMMIT_AND_FETCH_REQ will release these references minus those for
154	* any buffers registered on the io daemon task.
155	*/
156	#define UBLK_REFCOUNT_INIT (REFCOUNT_MAX / 2)
157
158	union ublk_io_buf {
159	__u64 addr;
160	struct ublk_auto_buf_reg auto_reg;
161	};
162
163	struct ublk_io {
164	union ublk_io_buf buf;
165	unsigned int flags;
166	int res;
167
168	union {
169	/* valid if UBLK_IO_FLAG_ACTIVE is set */
170	struct io_uring_cmd *cmd;
171	/* valid if UBLK_IO_FLAG_OWNED_BY_SRV is set */
172	struct request *req;
173	};
174
175	struct task_struct *task;
176
177	/*
178	* The number of uses of this I/O by the ublk server
179	* if user copy or zero copy are enabled:
180	* - UBLK_REFCOUNT_INIT from dispatch to the server
181	* until UBLK_IO_COMMIT_AND_FETCH_REQ
182	* - 1 for each inflight ublk_ch_{read,write}_iter() call
183	* - 1 for each io_uring registered buffer not registered on task
184	* The I/O can only be completed once all references are dropped.
185	* User copy and buffer registration operations are only permitted
186	* if the reference count is nonzero.
187	*/
188	refcount_t ref;
189	/* Count of buffers registered on task and not yet unregistered */
190	unsigned task_registered_buffers;
191
192	void *buf_ctx_handle;
193	} ____cacheline_aligned_in_smp;
194
195	struct ublk_queue {
196	int q_id;
197	int q_depth;
198
199	unsigned long flags;
200	struct ublksrv_io_desc *io_cmd_buf;
201
202	bool force_abort;
203	bool canceling;
204	bool fail_io; /* copy of dev->state == UBLK_S_DEV_FAIL_IO */
205	spinlock_t cancel_lock;
206	struct ublk_device *dev;
207	struct ublk_io ios[] __counted_by(q_depth);
208	};
209
210	struct ublk_device {
211	struct gendisk *ub_disk;
212
213	struct ublksrv_ctrl_dev_info dev_info;
214
215	struct blk_mq_tag_set tag_set;
216
217	struct cdev cdev;
218	struct device cdev_dev;
219
220	#define UB_STATE_OPEN 0
221	#define UB_STATE_USED 1
222	#define UB_STATE_DELETED 2
223	unsigned long state;
224	int ub_number;
225
226	struct mutex mutex;
227
228	spinlock_t lock;
229	struct mm_struct *mm;
230
231	struct ublk_params params;
232
233	struct completion completion;
234	u32 nr_io_ready;
235	bool unprivileged_daemons;
236	struct mutex cancel_mutex;
237	bool canceling;
238	pid_t ublksrv_tgid;
239	struct delayed_work exit_work;
240	struct work_struct partition_scan_work;
241
242	struct ublk_queue *queues[];
243	};
244
245	/* header of ublk_params */
246	struct ublk_params_header {
247	__u32 len;
248	__u32 types;
249	};
250
251	static void ublk_io_release(void *priv);
252	static void ublk_stop_dev_unlocked(struct ublk_device *ub);
253	static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq);
254	static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,
255	u16 q_id, u16 tag, struct ublk_io *io, size_t offset);
256	static inline unsigned int ublk_req_build_flags(struct request *req);
257
258	static inline struct ublksrv_io_desc *
259	ublk_get_iod(const struct ublk_queue *ubq, unsigned tag)
260	{
261	return &ubq->io_cmd_buf[tag];
262	}
263
264	static inline bool ublk_dev_is_zoned(const struct ublk_device *ub)
265	{
266	return ub->dev_info.flags & UBLK_F_ZONED;
267	}
268
269	static inline bool ublk_queue_is_zoned(const struct ublk_queue *ubq)
270	{
271	return ubq->flags & UBLK_F_ZONED;
272	}
273
274	#ifdef CONFIG_BLK_DEV_ZONED
275
276	struct ublk_zoned_report_desc {
277	__u64 sector;
278	__u32 operation;
279	__u32 nr_zones;
280	};
281
282	static DEFINE_XARRAY(ublk_zoned_report_descs);
283
284	static int ublk_zoned_insert_report_desc(const struct request *req,
285	struct ublk_zoned_report_desc *desc)
286	{
287	return xa_insert(xa: &ublk_zoned_report_descs, index: (unsigned long)req,
288	entry: desc, GFP_KERNEL);
289	}
290
291	static struct ublk_zoned_report_desc *ublk_zoned_erase_report_desc(
292	const struct request *req)
293	{
294	return xa_erase(&ublk_zoned_report_descs, index: (unsigned long)req);
295	}
296
297	static struct ublk_zoned_report_desc *ublk_zoned_get_report_desc(
298	const struct request *req)
299	{
300	return xa_load(&ublk_zoned_report_descs, index: (unsigned long)req);
301	}
302
303	static int ublk_get_nr_zones(const struct ublk_device *ub)
304	{
305	const struct ublk_param_basic *p = &ub->params.basic;
306
307	/* Zone size is a power of 2 */
308	return p->dev_sectors >> ilog2(p->chunk_sectors);
309	}
310
311	static int ublk_revalidate_disk_zones(struct ublk_device *ub)
312	{
313	return blk_revalidate_disk_zones(disk: ub->ub_disk);
314	}
315
316	static int ublk_dev_param_zoned_validate(const struct ublk_device *ub)
317	{
318	const struct ublk_param_zoned *p = &ub->params.zoned;
319	int nr_zones;
320
321	if (!ublk_dev_is_zoned(ub))
322	return -EINVAL;
323
324	if (!p->max_zone_append_sectors)
325	return -EINVAL;
326
327	nr_zones = ublk_get_nr_zones(ub);
328
329	if (p->max_active_zones > nr_zones)
330	return -EINVAL;
331
332	if (p->max_open_zones > nr_zones)
333	return -EINVAL;
334
335	return 0;
336	}
337
338	static void ublk_dev_param_zoned_apply(struct ublk_device *ub)
339	{
340	ub->ub_disk->nr_zones = ublk_get_nr_zones(ub);
341	}
342
343	/* Based on virtblk_alloc_report_buffer */
344	static void *ublk_alloc_report_buffer(struct ublk_device *ublk,
345	unsigned int nr_zones, size_t *buflen)
346	{
347	struct request_queue *q = ublk->ub_disk->queue;
348	size_t bufsize;
349	void *buf;
350
351	nr_zones = min_t(unsigned int, nr_zones,
352	ublk->ub_disk->nr_zones);
353
354	bufsize = nr_zones * sizeof(struct blk_zone);
355	bufsize =
356	min_t(size_t, bufsize, queue_max_hw_sectors(q) << SECTOR_SHIFT);
357
358	while (bufsize >= sizeof(struct blk_zone)) {
359	buf = kvmalloc(bufsize, GFP_KERNEL | __GFP_NORETRY);
360	if (buf) {
361	*buflen = bufsize;
362	return buf;
363	}
364	bufsize >>= 1;
365	}
366
367	*buflen = 0;
368	return NULL;
369	}
370
371	static int ublk_report_zones(struct gendisk *disk, sector_t sector,
372	unsigned int nr_zones, struct blk_report_zones_args *args)
373	{
374	struct ublk_device *ub = disk->private_data;
375	unsigned int zone_size_sectors = disk->queue->limits.chunk_sectors;
376	unsigned int first_zone = sector >> ilog2(zone_size_sectors);
377	unsigned int done_zones = 0;
378	unsigned int max_zones_per_request;
379	int ret;
380	struct blk_zone *buffer;
381	size_t buffer_length;
382
383	nr_zones = min_t(unsigned int, ub->ub_disk->nr_zones - first_zone,
384	nr_zones);
385
386	buffer = ublk_alloc_report_buffer(ublk: ub, nr_zones, buflen: &buffer_length);
387	if (!buffer)
388	return -ENOMEM;
389
390	max_zones_per_request = buffer_length / sizeof(struct blk_zone);
391
392	while (done_zones < nr_zones) {
393	unsigned int remaining_zones = nr_zones - done_zones;
394	unsigned int zones_in_request =
395	min_t(unsigned int, remaining_zones, max_zones_per_request);
396	struct request *req;
397	struct ublk_zoned_report_desc desc;
398	blk_status_t status;
399
400	memset(buffer, 0, buffer_length);
401
402	req = blk_mq_alloc_request(q: disk->queue, opf: REQ_OP_DRV_IN, flags: 0);
403	if (IS_ERR(ptr: req)) {
404	ret = PTR_ERR(ptr: req);
405	goto out;
406	}
407
408	desc.operation = UBLK_IO_OP_REPORT_ZONES;
409	desc.sector = sector;
410	desc.nr_zones = zones_in_request;
411	ret = ublk_zoned_insert_report_desc(req, desc: &desc);
412	if (ret)
413	goto free_req;
414
415	ret = blk_rq_map_kern(rq: req, kbuf: buffer, len: buffer_length, GFP_KERNEL);
416	if (ret)
417	goto erase_desc;
418
419	status = blk_execute_rq(rq: req, at_head: 0);
420	ret = blk_status_to_errno(status);
421	erase_desc:
422	ublk_zoned_erase_report_desc(req);
423	free_req:
424	blk_mq_free_request(rq: req);
425	if (ret)
426	goto out;
427
428	for (unsigned int i = 0; i < zones_in_request; i++) {
429	struct blk_zone *zone = buffer + i;
430
431	/* A zero length zone means no more zones in this response */
432	if (!zone->len)
433	break;
434
435	ret = disk_report_zone(disk, zone, idx: i, args);
436	if (ret)
437	goto out;
438
439	done_zones++;
440	sector += zone_size_sectors;
441
442	}
443	}
444
445	ret = done_zones;
446
447	out:
448	kvfree(addr: buffer);
449	return ret;
450	}
451
452	static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
453	struct request *req)
454	{
455	struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag: req->tag);
456	struct ublk_io *io = &ubq->ios[req->tag];
457	struct ublk_zoned_report_desc *desc;
458	u32 ublk_op;
459
460	switch (req_op(req)) {
461	case REQ_OP_ZONE_OPEN:
462	ublk_op = UBLK_IO_OP_ZONE_OPEN;
463	break;
464	case REQ_OP_ZONE_CLOSE:
465	ublk_op = UBLK_IO_OP_ZONE_CLOSE;
466	break;
467	case REQ_OP_ZONE_FINISH:
468	ublk_op = UBLK_IO_OP_ZONE_FINISH;
469	break;
470	case REQ_OP_ZONE_RESET:
471	ublk_op = UBLK_IO_OP_ZONE_RESET;
472	break;
473	case REQ_OP_ZONE_APPEND:
474	ublk_op = UBLK_IO_OP_ZONE_APPEND;
475	break;
476	case REQ_OP_ZONE_RESET_ALL:
477	ublk_op = UBLK_IO_OP_ZONE_RESET_ALL;
478	break;
479	case REQ_OP_DRV_IN:
480	desc = ublk_zoned_get_report_desc(req);
481	if (!desc)
482	return BLK_STS_IOERR;
483	ublk_op = desc->operation;
484	switch (ublk_op) {
485	case UBLK_IO_OP_REPORT_ZONES:
486	iod->op_flags = ublk_op | ublk_req_build_flags(req);
487	iod->nr_zones = desc->nr_zones;
488	iod->start_sector = desc->sector;
489	return BLK_STS_OK;
490	default:
491	return BLK_STS_IOERR;
492	}
493	case REQ_OP_DRV_OUT:
494	/* We do not support drv_out */
495	return BLK_STS_NOTSUPP;
496	default:
497	return BLK_STS_IOERR;
498	}
499
500	iod->op_flags = ublk_op | ublk_req_build_flags(req);
501	iod->nr_sectors = blk_rq_sectors(rq: req);
502	iod->start_sector = blk_rq_pos(rq: req);
503	iod->addr = io->buf.addr;
504
505	return BLK_STS_OK;
506	}
507
508	#else
509
510	#define ublk_report_zones (NULL)
511
512	static int ublk_dev_param_zoned_validate(const struct ublk_device *ub)
513	{
514	return -EOPNOTSUPP;
515	}
516
517	static void ublk_dev_param_zoned_apply(struct ublk_device *ub)
518	{
519	}
520
521	static int ublk_revalidate_disk_zones(struct ublk_device *ub)
522	{
523	return 0;
524	}
525
526	static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
527	struct request *req)
528	{
529	return BLK_STS_NOTSUPP;
530	}
531
532	#endif
533
534	static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
535	bool need_map);
536
537	static dev_t ublk_chr_devt;
538	static const struct class ublk_chr_class = {
539	.name = "ublk-char",
540	};
541
542	static DEFINE_IDR(ublk_index_idr);
543	static DEFINE_SPINLOCK(ublk_idr_lock);
544	static wait_queue_head_t ublk_idr_wq; /* wait until one idr is freed */
545
546	static DEFINE_MUTEX(ublk_ctl_mutex);
547
548
549	#define UBLK_MAX_UBLKS UBLK_MINORS
550
551	/*
552	* Max unprivileged ublk devices allowed to add
553	*
554	* It can be extended to one per-user limit in future or even controlled
555	* by cgroup.
556	*/
557	static unsigned int unprivileged_ublks_max = 64;
558	static unsigned int unprivileged_ublks_added; /* protected by ublk_ctl_mutex */
559
560	static struct miscdevice ublk_misc;
561
562	static inline unsigned ublk_pos_to_hwq(loff_t pos)
563	{
564	return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_QID_OFF) &
565	UBLK_QID_BITS_MASK;
566	}
567
568	static inline unsigned ublk_pos_to_buf_off(loff_t pos)
569	{
570	return (pos - UBLKSRV_IO_BUF_OFFSET) & UBLK_IO_BUF_BITS_MASK;
571	}
572
573	static inline unsigned ublk_pos_to_tag(loff_t pos)
574	{
575	return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_TAG_OFF) &
576	UBLK_TAG_BITS_MASK;
577	}
578
579	static void ublk_dev_param_basic_apply(struct ublk_device *ub)
580	{
581	const struct ublk_param_basic *p = &ub->params.basic;
582
583	if (p->attrs & UBLK_ATTR_READ_ONLY)
584	set_disk_ro(disk: ub->ub_disk, read_only: true);
585
586	set_capacity(disk: ub->ub_disk, size: p->dev_sectors);
587	}
588
589	static int ublk_validate_params(const struct ublk_device *ub)
590	{
591	/* basic param is the only one which must be set */
592	if (ub->params.types & UBLK_PARAM_TYPE_BASIC) {
593	const struct ublk_param_basic *p = &ub->params.basic;
594
595	if (p->logical_bs_shift > PAGE_SHIFT || p->logical_bs_shift < 9)
596	return -EINVAL;
597
598	if (p->logical_bs_shift > p->physical_bs_shift)
599	return -EINVAL;
600
601	if (p->max_sectors > (ub->dev_info.max_io_buf_bytes >> 9))
602	return -EINVAL;
603
604	if (ublk_dev_is_zoned(ub) && !p->chunk_sectors)
605	return -EINVAL;
606	} else
607	return -EINVAL;
608
609	if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) {
610	const struct ublk_param_discard *p = &ub->params.discard;
611
612	/* So far, only support single segment discard */
613	if (p->max_discard_sectors && p->max_discard_segments != 1)
614	return -EINVAL;
615
616	if (!p->discard_granularity)
617	return -EINVAL;
618	}
619
620	/* dev_t is read-only */
621	if (ub->params.types & UBLK_PARAM_TYPE_DEVT)
622	return -EINVAL;
623
624	if (ub->params.types & UBLK_PARAM_TYPE_ZONED)
625	return ublk_dev_param_zoned_validate(ub);
626	else if (ublk_dev_is_zoned(ub))
627	return -EINVAL;
628
629	if (ub->params.types & UBLK_PARAM_TYPE_DMA_ALIGN) {
630	const struct ublk_param_dma_align *p = &ub->params.dma;
631
632	if (p->alignment >= PAGE_SIZE)
633	return -EINVAL;
634
635	if (!is_power_of_2(n: p->alignment + 1))
636	return -EINVAL;
637	}
638
639	if (ub->params.types & UBLK_PARAM_TYPE_SEGMENT) {
640	const struct ublk_param_segment *p = &ub->params.seg;
641
642	if (!is_power_of_2(n: p->seg_boundary_mask + 1))
643	return -EINVAL;
644
645	if (p->seg_boundary_mask + 1 < UBLK_MIN_SEGMENT_SIZE)
646	return -EINVAL;
647	if (p->max_segment_size < UBLK_MIN_SEGMENT_SIZE)
648	return -EINVAL;
649	}
650
651	return 0;
652	}
653
654	static void ublk_apply_params(struct ublk_device *ub)
655	{
656	ublk_dev_param_basic_apply(ub);
657
658	if (ub->params.types & UBLK_PARAM_TYPE_ZONED)
659	ublk_dev_param_zoned_apply(ub);
660	}
661
662	static inline bool ublk_support_zero_copy(const struct ublk_queue *ubq)
663	{
664	return ubq->flags & UBLK_F_SUPPORT_ZERO_COPY;
665	}
666
667	static inline bool ublk_dev_support_zero_copy(const struct ublk_device *ub)
668	{
669	return ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY;
670	}
671
672	static inline bool ublk_support_auto_buf_reg(const struct ublk_queue *ubq)
673	{
674	return ubq->flags & UBLK_F_AUTO_BUF_REG;
675	}
676
677	static inline bool ublk_dev_support_auto_buf_reg(const struct ublk_device *ub)
678	{
679	return ub->dev_info.flags & UBLK_F_AUTO_BUF_REG;
680	}
681
682	static inline bool ublk_support_user_copy(const struct ublk_queue *ubq)
683	{
684	return ubq->flags & UBLK_F_USER_COPY;
685	}
686
687	static inline bool ublk_dev_support_user_copy(const struct ublk_device *ub)
688	{
689	return ub->dev_info.flags & UBLK_F_USER_COPY;
690	}
691
692	static inline bool ublk_need_map_io(const struct ublk_queue *ubq)
693	{
694	return !ublk_support_user_copy(ubq) && !ublk_support_zero_copy(ubq) &&
695	!ublk_support_auto_buf_reg(ubq);
696	}
697
698	static inline bool ublk_dev_need_map_io(const struct ublk_device *ub)
699	{
700	return !ublk_dev_support_user_copy(ub) &&
701	!ublk_dev_support_zero_copy(ub) &&
702	!ublk_dev_support_auto_buf_reg(ub);
703	}
704
705	static inline bool ublk_need_req_ref(const struct ublk_queue *ubq)
706	{
707	/*
708	* read()/write() is involved in user copy, so request reference
709	* has to be grabbed
710	*
711	* for zero copy, request buffer need to be registered to io_uring
712	* buffer table, so reference is needed
713	*
714	* For auto buffer register, ublk server still may issue
715	* UBLK_IO_COMMIT_AND_FETCH_REQ before one registered buffer is used up,
716	* so reference is required too.
717	*/
718	return ublk_support_user_copy(ubq) || ublk_support_zero_copy(ubq) ||
719	ublk_support_auto_buf_reg(ubq);
720	}
721
722	static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)
723	{
724	return ublk_dev_support_user_copy(ub) ||
725	ublk_dev_support_zero_copy(ub) ||
726	ublk_dev_support_auto_buf_reg(ub);
727	}
728
729	static inline void ublk_init_req_ref(const struct ublk_queue *ubq,
730	struct ublk_io *io)
731	{
732	if (ublk_need_req_ref(ubq))
733	refcount_set(r: &io->ref, UBLK_REFCOUNT_INIT);
734	}
735
736	static inline bool ublk_get_req_ref(struct ublk_io *io)
737	{
738	return refcount_inc_not_zero(r: &io->ref);
739	}
740
741	static inline void ublk_put_req_ref(struct ublk_io *io, struct request *req)
742	{
743	if (!refcount_dec_and_test(r: &io->ref))
744	return;
745
746	/* ublk_need_map_io() and ublk_need_req_ref() are mutually exclusive */
747	__ublk_complete_rq(req, io, need_map: false);
748	}
749
750	static inline bool ublk_sub_req_ref(struct ublk_io *io)
751	{
752	unsigned sub_refs = UBLK_REFCOUNT_INIT - io->task_registered_buffers;
753
754	io->task_registered_buffers = 0;
755	return refcount_sub_and_test(i: sub_refs, r: &io->ref);
756	}
757
758	static inline bool ublk_need_get_data(const struct ublk_queue *ubq)
759	{
760	return ubq->flags & UBLK_F_NEED_GET_DATA;
761	}
762
763	static inline bool ublk_dev_need_get_data(const struct ublk_device *ub)
764	{
765	return ub->dev_info.flags & UBLK_F_NEED_GET_DATA;
766	}
767
768	/* Called in slow path only, keep it noinline for trace purpose */
769	static noinline struct ublk_device *ublk_get_device(struct ublk_device *ub)
770	{
771	if (kobject_get_unless_zero(kobj: &ub->cdev_dev.kobj))
772	return ub;
773	return NULL;
774	}
775
776	/* Called in slow path only, keep it noinline for trace purpose */
777	static noinline void ublk_put_device(struct ublk_device *ub)
778	{
779	put_device(dev: &ub->cdev_dev);
780	}
781
782	static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev,
783	int qid)
784	{
785	return dev->queues[qid];
786	}
787
788	static inline bool ublk_rq_has_data(const struct request *rq)
789	{
790	return bio_has_data(bio: rq->bio);
791	}
792
793	static inline struct ublksrv_io_desc *
794	ublk_queue_cmd_buf(struct ublk_device *ub, int q_id)
795	{
796	return ublk_get_queue(dev: ub, qid: q_id)->io_cmd_buf;
797	}
798
799	static inline int __ublk_queue_cmd_buf_size(int depth)
800	{
801	return round_up(depth * sizeof(struct ublksrv_io_desc), PAGE_SIZE);
802	}
803
804	static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub)
805	{
806	return __ublk_queue_cmd_buf_size(depth: ub->dev_info.queue_depth);
807	}
808
809	static int ublk_max_cmd_buf_size(void)
810	{
811	return __ublk_queue_cmd_buf_size(UBLK_MAX_QUEUE_DEPTH);
812	}
813
814	/*
815	* Should I/O outstanding to the ublk server when it exits be reissued?
816	* If not, outstanding I/O will get errors.
817	*/
818	static inline bool ublk_nosrv_should_reissue_outstanding(struct ublk_device *ub)
819	{
820	return (ub->dev_info.flags & UBLK_F_USER_RECOVERY) &&
821	(ub->dev_info.flags & UBLK_F_USER_RECOVERY_REISSUE);
822	}
823
824	/*
825	* Should I/O issued while there is no ublk server queue? If not, I/O
826	* issued while there is no ublk server will get errors.
827	*/
828	static inline bool ublk_nosrv_dev_should_queue_io(struct ublk_device *ub)
829	{
830	return (ub->dev_info.flags & UBLK_F_USER_RECOVERY) &&
831	!(ub->dev_info.flags & UBLK_F_USER_RECOVERY_FAIL_IO);
832	}
833
834	/*
835	* Same as ublk_nosrv_dev_should_queue_io, but uses a queue-local copy
836	* of the device flags for smaller cache footprint - better for fast
837	* paths.
838	*/
839	static inline bool ublk_nosrv_should_queue_io(struct ublk_queue *ubq)
840	{
841	return (ubq->flags & UBLK_F_USER_RECOVERY) &&
842	!(ubq->flags & UBLK_F_USER_RECOVERY_FAIL_IO);
843	}
844
845	/*
846	* Should ublk devices be stopped (i.e. no recovery possible) when the
847	* ublk server exits? If not, devices can be used again by a future
848	* incarnation of a ublk server via the start_recovery/end_recovery
849	* commands.
850	*/
851	static inline bool ublk_nosrv_should_stop_dev(struct ublk_device *ub)
852	{
853	return !(ub->dev_info.flags & UBLK_F_USER_RECOVERY);
854	}
855
856	static inline bool ublk_dev_in_recoverable_state(struct ublk_device *ub)
857	{
858	return ub->dev_info.state == UBLK_S_DEV_QUIESCED ||
859	ub->dev_info.state == UBLK_S_DEV_FAIL_IO;
860	}
861
862	static void ublk_free_disk(struct gendisk *disk)
863	{
864	struct ublk_device *ub = disk->private_data;
865
866	clear_bit(UB_STATE_USED, addr: &ub->state);
867	ublk_put_device(ub);
868	}
869
870	static void ublk_store_owner_uid_gid(unsigned int *owner_uid,
871	unsigned int *owner_gid)
872	{
873	kuid_t uid;
874	kgid_t gid;
875
876	current_uid_gid(&uid, &gid);
877
878	*owner_uid = from_kuid(to: &init_user_ns, uid);
879	*owner_gid = from_kgid(to: &init_user_ns, gid);
880	}
881
882	static int ublk_open(struct gendisk *disk, blk_mode_t mode)
883	{
884	struct ublk_device *ub = disk->private_data;
885
886	if (capable(CAP_SYS_ADMIN))
887	return 0;
888
889	/*
890	* If it is one unprivileged device, only owner can open
891	* the disk. Otherwise it could be one trap made by one
892	* evil user who grants this disk's privileges to other
893	* users deliberately.
894	*
895	* This way is reasonable too given anyone can create
896	* unprivileged device, and no need other's grant.
897	*/
898	if (ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV) {
899	unsigned int curr_uid, curr_gid;
900
901	ublk_store_owner_uid_gid(owner_uid: &curr_uid, owner_gid: &curr_gid);
902
903	if (curr_uid != ub->dev_info.owner_uid || curr_gid !=
904	ub->dev_info.owner_gid)
905	return -EPERM;
906	}
907
908	return 0;
909	}
910
911	static const struct block_device_operations ub_fops = {
912	.owner = THIS_MODULE,
913	.open = ublk_open,
914	.free_disk = ublk_free_disk,
915	.report_zones = ublk_report_zones,
916	};
917
918	/*
919	* Copy data between request pages and io_iter, and 'offset'
920	* is the start point of linear offset of request.
921	*/
922	static size_t ublk_copy_user_pages(const struct request *req,
923	unsigned offset, struct iov_iter *uiter, int dir)
924	{
925	struct req_iterator iter;
926	struct bio_vec bv;
927	size_t done = 0;
928
929	rq_for_each_segment(bv, req, iter) {
930	unsigned len;
931	void *bv_buf;
932	size_t copied;
933
934	if (offset >= bv.bv_len) {
935	offset -= bv.bv_len;
936	continue;
937	}
938
939	len = bv.bv_len - offset;
940	bv_buf = kmap_local_page(page: bv.bv_page) + bv.bv_offset + offset;
941	if (dir == ITER_DEST)
942	copied = copy_to_iter(addr: bv_buf, bytes: len, i: uiter);
943	else
944	copied = copy_from_iter(addr: bv_buf, bytes: len, i: uiter);
945
946	kunmap_local(bv_buf);
947
948	done += copied;
949	if (copied < len)
950	break;
951
952	offset = 0;
953	}
954	return done;
955	}
956
957	static inline bool ublk_need_map_req(const struct request *req)
958	{
959	return ublk_rq_has_data(rq: req) && req_op(req) == REQ_OP_WRITE;
960	}
961
962	static inline bool ublk_need_unmap_req(const struct request *req)
963	{
964	return ublk_rq_has_data(rq: req) &&
965	(req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_DRV_IN);
966	}
967
968	static unsigned int ublk_map_io(const struct ublk_queue *ubq,
969	const struct request *req,
970	const struct ublk_io *io)
971	{
972	const unsigned int rq_bytes = blk_rq_bytes(rq: req);
973
974	if (!ublk_need_map_io(ubq))
975	return rq_bytes;
976
977	/*
978	* no zero copy, we delay copy WRITE request data into ublksrv
979	* context and the big benefit is that pinning pages in current
980	* context is pretty fast, see ublk_pin_user_pages
981	*/
982	if (ublk_need_map_req(req)) {
983	struct iov_iter iter;
984	const int dir = ITER_DEST;
985
986	import_ubuf(type: dir, u64_to_user_ptr(io->buf.addr), len: rq_bytes, i: &iter);
987	return ublk_copy_user_pages(req, offset: 0, uiter: &iter, dir);
988	}
989	return rq_bytes;
990	}
991
992	static unsigned int ublk_unmap_io(bool need_map,
993	const struct request *req,
994	const struct ublk_io *io)
995	{
996	const unsigned int rq_bytes = blk_rq_bytes(rq: req);
997
998	if (!need_map)
999	return rq_bytes;
1000
1001	if (ublk_need_unmap_req(req)) {
1002	struct iov_iter iter;
1003	const int dir = ITER_SOURCE;
1004
1005	WARN_ON_ONCE(io->res > rq_bytes);
1006
1007	import_ubuf(type: dir, u64_to_user_ptr(io->buf.addr), len: io->res, i: &iter);
1008	return ublk_copy_user_pages(req, offset: 0, uiter: &iter, dir);
1009	}
1010	return rq_bytes;
1011	}
1012
1013	static inline unsigned int ublk_req_build_flags(struct request *req)
1014	{
1015	unsigned flags = 0;
1016
1017	if (req->cmd_flags & REQ_FAILFAST_DEV)
1018	flags |= UBLK_IO_F_FAILFAST_DEV;
1019
1020	if (req->cmd_flags & REQ_FAILFAST_TRANSPORT)
1021	flags |= UBLK_IO_F_FAILFAST_TRANSPORT;
1022
1023	if (req->cmd_flags & REQ_FAILFAST_DRIVER)
1024	flags |= UBLK_IO_F_FAILFAST_DRIVER;
1025
1026	if (req->cmd_flags & REQ_META)
1027	flags |= UBLK_IO_F_META;
1028
1029	if (req->cmd_flags & REQ_FUA)
1030	flags |= UBLK_IO_F_FUA;
1031
1032	if (req->cmd_flags & REQ_NOUNMAP)
1033	flags |= UBLK_IO_F_NOUNMAP;
1034
1035	if (req->cmd_flags & REQ_SWAP)
1036	flags |= UBLK_IO_F_SWAP;
1037
1038	return flags;
1039	}
1040
1041	static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req)
1042	{
1043	struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag: req->tag);
1044	struct ublk_io *io = &ubq->ios[req->tag];
1045	u32 ublk_op;
1046
1047	switch (req_op(req)) {
1048	case REQ_OP_READ:
1049	ublk_op = UBLK_IO_OP_READ;
1050	break;
1051	case REQ_OP_WRITE:
1052	ublk_op = UBLK_IO_OP_WRITE;
1053	break;
1054	case REQ_OP_FLUSH:
1055	ublk_op = UBLK_IO_OP_FLUSH;
1056	break;
1057	case REQ_OP_DISCARD:
1058	ublk_op = UBLK_IO_OP_DISCARD;
1059	break;
1060	case REQ_OP_WRITE_ZEROES:
1061	ublk_op = UBLK_IO_OP_WRITE_ZEROES;
1062	break;
1063	default:
1064	if (ublk_queue_is_zoned(ubq))
1065	return ublk_setup_iod_zoned(ubq, req);
1066	return BLK_STS_IOERR;
1067	}
1068
1069	/* need to translate since kernel may change */
1070	iod->op_flags = ublk_op | ublk_req_build_flags(req);
1071	iod->nr_sectors = blk_rq_sectors(rq: req);
1072	iod->start_sector = blk_rq_pos(rq: req);
1073	iod->addr = io->buf.addr;
1074
1075	return BLK_STS_OK;
1076	}
1077
1078	static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu(
1079	struct io_uring_cmd *ioucmd)
1080	{
1081	return io_uring_cmd_to_pdu(ioucmd, struct ublk_uring_cmd_pdu);
1082	}
1083
1084	static void ublk_end_request(struct request *req, blk_status_t error)
1085	{
1086	local_bh_disable();
1087	blk_mq_end_request(rq: req, error);
1088	local_bh_enable();
1089	}
1090
1091	/* todo: handle partial completion */
1092	static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
1093	bool need_map)
1094	{
1095	unsigned int unmapped_bytes;
1096	blk_status_t res = BLK_STS_OK;
1097	bool requeue;
1098
1099	/* failed read IO if nothing is read */
1100	if (!io->res && req_op(req) == REQ_OP_READ)
1101	io->res = -EIO;
1102
1103	if (io->res < 0) {
1104	res = errno_to_blk_status(errno: io->res);
1105	goto exit;
1106	}
1107
1108	/*
1109	* FLUSH, DISCARD or WRITE_ZEROES usually won't return bytes returned, so end them
1110	* directly.
1111	*
1112	* Both the two needn't unmap.
1113	*/
1114	if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE &&
1115	req_op(req) != REQ_OP_DRV_IN)
1116	goto exit;
1117
1118	/* for READ request, writing data in iod->addr to rq buffers */
1119	unmapped_bytes = ublk_unmap_io(need_map, req, io);
1120
1121	/*
1122	* Extremely impossible since we got data filled in just before
1123	*
1124	* Re-read simply for this unlikely case.
1125	*/
1126	if (unlikely(unmapped_bytes < io->res))
1127	io->res = unmapped_bytes;
1128
1129	/*
1130	* Run bio->bi_end_io() with softirqs disabled. If the final fput
1131	* happens off this path, then that will prevent ublk's blkdev_release()
1132	* from being called on current's task work, see fput() implementation.
1133	*
1134	* Otherwise, ublk server may not provide forward progress in case of
1135	* reading the partition table from bdev_open() with disk->open_mutex
1136	* held, and causes dead lock as we could already be holding
1137	* disk->open_mutex here.
1138	*
1139	* Preferably we would not be doing IO with a mutex held that is also
1140	* used for release, but this work-around will suffice for now.
1141	*/
1142	local_bh_disable();
1143	requeue = blk_update_request(rq: req, BLK_STS_OK, nr_bytes: io->res);
1144	local_bh_enable();
1145	if (requeue)
1146	blk_mq_requeue_request(rq: req, kick_requeue_list: true);
1147	else if (likely(!blk_should_fake_timeout(req->q)))
1148	__blk_mq_end_request(rq: req, BLK_STS_OK);
1149
1150	return;
1151	exit:
1152	ublk_end_request(req, error: res);
1153	}
1154
1155	static struct io_uring_cmd *__ublk_prep_compl_io_cmd(struct ublk_io *io,
1156	struct request *req)
1157	{
1158	/* read cmd first because req will overwrite it */
1159	struct io_uring_cmd *cmd = io->cmd;
1160
1161	/* mark this cmd owned by ublksrv */
1162	io->flags |= UBLK_IO_FLAG_OWNED_BY_SRV;
1163
1164	/*
1165	* clear ACTIVE since we are done with this sqe/cmd slot
1166	* We can only accept io cmd in case of being not active.
1167	*/
1168	io->flags &= ~UBLK_IO_FLAG_ACTIVE;
1169
1170	io->req = req;
1171	return cmd;
1172	}
1173
1174	static void ublk_complete_io_cmd(struct ublk_io *io, struct request *req,
1175	int res, unsigned issue_flags)
1176	{
1177	struct io_uring_cmd *cmd = __ublk_prep_compl_io_cmd(io, req);
1178
1179	/* tell ublksrv one io request is coming */
1180	io_uring_cmd_done(ioucmd: cmd, ret: res, issue_flags);
1181	}
1182
1183	#define UBLK_REQUEUE_DELAY_MS 3
1184
1185	static inline void __ublk_abort_rq(struct ublk_queue *ubq,
1186	struct request *rq)
1187	{
1188	/* We cannot process this rq so just requeue it. */
1189	if (ublk_nosrv_dev_should_queue_io(ub: ubq->dev))
1190	blk_mq_requeue_request(rq, kick_requeue_list: false);
1191	else
1192	ublk_end_request(req: rq, BLK_STS_IOERR);
1193	}
1194
1195	static void
1196	ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, unsigned tag)
1197	{
1198	struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag);
1199
1200	iod->op_flags |= UBLK_IO_F_NEED_REG_BUF;
1201	}
1202
1203	enum auto_buf_reg_res {
1204	AUTO_BUF_REG_FAIL,
1205	AUTO_BUF_REG_FALLBACK,
1206	AUTO_BUF_REG_OK,
1207	};
1208
1209	static void ublk_prep_auto_buf_reg_io(const struct ublk_queue *ubq,
1210	struct request *req, struct ublk_io *io,
1211	struct io_uring_cmd *cmd,
1212	enum auto_buf_reg_res res)
1213	{
1214	if (res == AUTO_BUF_REG_OK) {
1215	io->task_registered_buffers = 1;
1216	io->buf_ctx_handle = io_uring_cmd_ctx_handle(cmd);
1217	io->flags |= UBLK_IO_FLAG_AUTO_BUF_REG;
1218	}
1219	ublk_init_req_ref(ubq, io);
1220	__ublk_prep_compl_io_cmd(io, req);
1221	}
1222
1223	static enum auto_buf_reg_res
1224	__ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,
1225	struct ublk_io *io, struct io_uring_cmd *cmd,
1226	unsigned int issue_flags)
1227	{
1228	int ret;
1229
1230	ret = io_buffer_register_bvec(cmd, rq: req, release: ublk_io_release,
1231	index: io->buf.auto_reg.index, issue_flags);
1232	if (ret) {
1233	if (io->buf.auto_reg.flags & UBLK_AUTO_BUF_REG_FALLBACK) {
1234	ublk_auto_buf_reg_fallback(ubq, tag: req->tag);
1235	return AUTO_BUF_REG_FALLBACK;
1236	}
1237	ublk_end_request(req, BLK_STS_IOERR);
1238	return AUTO_BUF_REG_FAIL;
1239	}
1240
1241	return AUTO_BUF_REG_OK;
1242	}
1243
1244	static void ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,
1245	struct ublk_io *io, struct io_uring_cmd *cmd,
1246	unsigned int issue_flags)
1247	{
1248	enum auto_buf_reg_res res = __ublk_do_auto_buf_reg(ubq, req, io, cmd,
1249	issue_flags);
1250
1251	if (res != AUTO_BUF_REG_FAIL) {
1252	ublk_prep_auto_buf_reg_io(ubq, req, io, cmd, res);
1253	io_uring_cmd_done(ioucmd: cmd, UBLK_IO_RES_OK, issue_flags);
1254	}
1255	}
1256
1257	static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req,
1258	struct ublk_io *io)
1259	{
1260	unsigned mapped_bytes = ublk_map_io(ubq, req, io);
1261
1262	/* partially mapped, update io descriptor */
1263	if (unlikely(mapped_bytes != blk_rq_bytes(req))) {
1264	/*
1265	* Nothing mapped, retry until we succeed.
1266	*
1267	* We may never succeed in mapping any bytes here because
1268	* of OOM. TODO: reserve one buffer with single page pinned
1269	* for providing forward progress guarantee.
1270	*/
1271	if (unlikely(!mapped_bytes)) {
1272	blk_mq_requeue_request(rq: req, kick_requeue_list: false);
1273	blk_mq_delay_kick_requeue_list(q: req->q,
1274	UBLK_REQUEUE_DELAY_MS);
1275	return false;
1276	}
1277
1278	ublk_get_iod(ubq, tag: req->tag)->nr_sectors =
1279	mapped_bytes >> 9;
1280	}
1281
1282	return true;
1283	}
1284
1285	static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req)
1286	{
1287	unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
1288	int tag = req->tag;
1289	struct ublk_io *io = &ubq->ios[tag];
1290
1291	pr_devel("%s: complete: qid %d tag %d io_flags %x addr %llx\n",
1292	__func__, ubq->q_id, req->tag, io->flags,
1293	ublk_get_iod(ubq, req->tag)->addr);
1294
1295	/*
1296	* Task is exiting if either:
1297	*
1298	* (1) current != io->task.
1299	* io_uring_cmd_complete_in_task() tries to run task_work
1300	* in a workqueue if cmd's task is PF_EXITING.
1301	*
1302	* (2) current->flags & PF_EXITING.
1303	*/
1304	if (unlikely(current != io->task || current->flags & PF_EXITING)) {
1305	__ublk_abort_rq(ubq, rq: req);
1306	return;
1307	}
1308
1309	if (ublk_need_get_data(ubq) && ublk_need_map_req(req)) {
1310	/*
1311	* We have not handled UBLK_IO_NEED_GET_DATA command yet,
1312	* so immediately pass UBLK_IO_RES_NEED_GET_DATA to ublksrv
1313	* and notify it.
1314	*/
1315	io->flags |= UBLK_IO_FLAG_NEED_GET_DATA;
1316	pr_devel("%s: need get data. qid %d tag %d io_flags %x\n",
1317	__func__, ubq->q_id, req->tag, io->flags);
1318	ublk_complete_io_cmd(io, req, UBLK_IO_RES_NEED_GET_DATA,
1319	issue_flags);
1320	return;
1321	}
1322
1323	if (!ublk_start_io(ubq, req, io))
1324	return;
1325
1326	if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(rq: req)) {
1327	ublk_do_auto_buf_reg(ubq, req, io, cmd: io->cmd, issue_flags);
1328	} else {
1329	ublk_init_req_ref(ubq, io);
1330	ublk_complete_io_cmd(io, req, UBLK_IO_RES_OK, issue_flags);
1331	}
1332	}
1333
1334	static void ublk_cmd_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw)
1335	{
1336	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
1337	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(ioucmd: cmd);
1338	struct ublk_queue *ubq = pdu->ubq;
1339
1340	ublk_dispatch_req(ubq, req: pdu->req);
1341	}
1342
1343	static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
1344	{
1345	struct io_uring_cmd *cmd = ubq->ios[rq->tag].cmd;
1346	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(ioucmd: cmd);
1347
1348	pdu->req = rq;
1349	io_uring_cmd_complete_in_task(ioucmd: cmd, task_work_cb: ublk_cmd_tw_cb);
1350	}
1351
1352	static void ublk_cmd_list_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw)
1353	{
1354	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
1355	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(ioucmd: cmd);
1356	struct request *rq = pdu->req_list;
1357	struct request *next;
1358
1359	do {
1360	next = rq->rq_next;
1361	rq->rq_next = NULL;
1362	ublk_dispatch_req(ubq: rq->mq_hctx->driver_data, req: rq);
1363	rq = next;
1364	} while (rq);
1365	}
1366
1367	static void ublk_queue_cmd_list(struct ublk_io *io, struct rq_list *l)
1368	{
1369	struct io_uring_cmd *cmd = io->cmd;
1370	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(ioucmd: cmd);
1371
1372	pdu->req_list = rq_list_peek(rl: l);
1373	rq_list_init(rl: l);
1374	io_uring_cmd_complete_in_task(ioucmd: cmd, task_work_cb: ublk_cmd_list_tw_cb);
1375	}
1376
1377	static enum blk_eh_timer_return ublk_timeout(struct request *rq)
1378	{
1379	struct ublk_queue *ubq = rq->mq_hctx->driver_data;
1380	pid_t tgid = ubq->dev->ublksrv_tgid;
1381	struct task_struct *p;
1382	struct pid *pid;
1383
1384	if (!(ubq->flags & UBLK_F_UNPRIVILEGED_DEV))
1385	return BLK_EH_RESET_TIMER;
1386
1387	if (unlikely(!tgid))
1388	return BLK_EH_RESET_TIMER;
1389
1390	rcu_read_lock();
1391	pid = find_vpid(nr: tgid);
1392	p = pid_task(pid, PIDTYPE_PID);
1393	if (p)
1394	send_sig(SIGKILL, p, 0);
1395	rcu_read_unlock();
1396	return BLK_EH_DONE;
1397	}
1398
1399	static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq,
1400	bool check_cancel)
1401	{
1402	blk_status_t res;
1403
1404	if (unlikely(READ_ONCE(ubq->fail_io)))
1405	return BLK_STS_TARGET;
1406
1407	/* With recovery feature enabled, force_abort is set in
1408	* ublk_stop_dev() before calling del_gendisk(). We have to
1409	* abort all requeued and new rqs here to let del_gendisk()
1410	* move on. Besides, we cannot not call io_uring_cmd_complete_in_task()
1411	* to avoid UAF on io_uring ctx.
1412	*
1413	* Note: force_abort is guaranteed to be seen because it is set
1414	* before request queue is unqiuesced.
1415	*/
1416	if (ublk_nosrv_should_queue_io(ubq) &&
1417	unlikely(READ_ONCE(ubq->force_abort)))
1418	return BLK_STS_IOERR;
1419
1420	if (check_cancel && unlikely(ubq->canceling))
1421	return BLK_STS_IOERR;
1422
1423	/* fill iod to slot in io cmd buffer */
1424	res = ublk_setup_iod(ubq, req: rq);
1425	if (unlikely(res != BLK_STS_OK))
1426	return BLK_STS_IOERR;
1427
1428	blk_mq_start_request(rq);
1429	return BLK_STS_OK;
1430	}
1431
1432	static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
1433	const struct blk_mq_queue_data *bd)
1434	{
1435	struct ublk_queue *ubq = hctx->driver_data;
1436	struct request *rq = bd->rq;
1437	blk_status_t res;
1438
1439	res = ublk_prep_req(ubq, rq, check_cancel: false);
1440	if (res != BLK_STS_OK)
1441	return res;
1442
1443	/*
1444	* ->canceling has to be handled after ->force_abort and ->fail_io
1445	* is dealt with, otherwise this request may not be failed in case
1446	* of recovery, and cause hang when deleting disk
1447	*/
1448	if (unlikely(ubq->canceling)) {
1449	__ublk_abort_rq(ubq, rq);
1450	return BLK_STS_OK;
1451	}
1452
1453	ublk_queue_cmd(ubq, rq);
1454	return BLK_STS_OK;
1455	}
1456
1457	static inline bool ublk_belong_to_same_batch(const struct ublk_io *io,
1458	const struct ublk_io *io2)
1459	{
1460	return (io_uring_cmd_ctx_handle(cmd: io->cmd) ==
1461	io_uring_cmd_ctx_handle(cmd: io2->cmd)) &&
1462	(io->task == io2->task);
1463	}
1464
1465	static void ublk_queue_rqs(struct rq_list *rqlist)
1466	{
1467	struct rq_list requeue_list = { };
1468	struct rq_list submit_list = { };
1469	struct ublk_io *io = NULL;
1470	struct request *req;
1471
1472	while ((req = rq_list_pop(rl: rqlist))) {
1473	struct ublk_queue *this_q = req->mq_hctx->driver_data;
1474	struct ublk_io *this_io = &this_q->ios[req->tag];
1475
1476	if (ublk_prep_req(ubq: this_q, rq: req, check_cancel: true) != BLK_STS_OK) {
1477	rq_list_add_tail(rl: &requeue_list, rq: req);
1478	continue;
1479	}
1480
1481	if (io && !ublk_belong_to_same_batch(io, io2: this_io) &&
1482	!rq_list_empty(rl: &submit_list))
1483	ublk_queue_cmd_list(io, l: &submit_list);
1484	io = this_io;
1485	rq_list_add_tail(rl: &submit_list, rq: req);
1486	}
1487
1488	if (!rq_list_empty(rl: &submit_list))
1489	ublk_queue_cmd_list(io, l: &submit_list);
1490	*rqlist = requeue_list;
1491	}
1492
1493	static int ublk_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
1494	unsigned int hctx_idx)
1495	{
1496	struct ublk_device *ub = driver_data;
1497	struct ublk_queue *ubq = ublk_get_queue(dev: ub, qid: hctx->queue_num);
1498
1499	hctx->driver_data = ubq;
1500	return 0;
1501	}
1502
1503	static const struct blk_mq_ops ublk_mq_ops = {
1504	.queue_rq = ublk_queue_rq,
1505	.queue_rqs = ublk_queue_rqs,
1506	.init_hctx = ublk_init_hctx,
1507	.timeout = ublk_timeout,
1508	};
1509
1510	static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq)
1511	{
1512	int i;
1513
1514	for (i = 0; i < ubq->q_depth; i++) {
1515	struct ublk_io *io = &ubq->ios[i];
1516
1517	/*
1518	* UBLK_IO_FLAG_CANCELED is kept for avoiding to touch
1519	* io->cmd
1520	*/
1521	io->flags &= UBLK_IO_FLAG_CANCELED;
1522	io->cmd = NULL;
1523	io->buf.addr = 0;
1524
1525	/*
1526	* old task is PF_EXITING, put it now
1527	*
1528	* It could be NULL in case of closing one quiesced
1529	* device.
1530	*/
1531	if (io->task) {
1532	put_task_struct(t: io->task);
1533	io->task = NULL;
1534	}
1535
1536	WARN_ON_ONCE(refcount_read(&io->ref));
1537	WARN_ON_ONCE(io->task_registered_buffers);
1538	}
1539	}
1540
1541	static int ublk_ch_open(struct inode *inode, struct file *filp)
1542	{
1543	struct ublk_device *ub = container_of(inode->i_cdev,
1544	struct ublk_device, cdev);
1545
1546	if (test_and_set_bit(UB_STATE_OPEN, addr: &ub->state))
1547	return -EBUSY;
1548	filp->private_data = ub;
1549	ub->ublksrv_tgid = current->tgid;
1550	return 0;
1551	}
1552
1553	static void ublk_reset_ch_dev(struct ublk_device *ub)
1554	{
1555	int i;
1556
1557	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1558	ublk_queue_reinit(ub, ubq: ublk_get_queue(dev: ub, qid: i));
1559
1560	/* set to NULL, otherwise new tasks cannot mmap io_cmd_buf */
1561	ub->mm = NULL;
1562	ub->nr_io_ready = 0;
1563	ub->unprivileged_daemons = false;
1564	ub->ublksrv_tgid = -1;
1565	}
1566
1567	static struct gendisk *ublk_get_disk(struct ublk_device *ub)
1568	{
1569	struct gendisk *disk;
1570
1571	spin_lock(lock: &ub->lock);
1572	disk = ub->ub_disk;
1573	if (disk)
1574	get_device(disk_to_dev(disk));
1575	spin_unlock(lock: &ub->lock);
1576
1577	return disk;
1578	}
1579
1580	static void ublk_put_disk(struct gendisk *disk)
1581	{
1582	if (disk)
1583	put_device(disk_to_dev(disk));
1584	}
1585
1586	static void ublk_partition_scan_work(struct work_struct *work)
1587	{
1588	struct ublk_device *ub =
1589	container_of(work, struct ublk_device, partition_scan_work);
1590	/* Hold disk reference to prevent UAF during concurrent teardown */
1591	struct gendisk *disk = ublk_get_disk(ub);
1592
1593	if (!disk)
1594	return;
1595
1596	if (WARN_ON_ONCE(!test_and_clear_bit(GD_SUPPRESS_PART_SCAN,
1597	&disk->state)))
1598	goto out;
1599
1600	mutex_lock(&disk->open_mutex);
1601	bdev_disk_changed(disk, invalidate: false);
1602	mutex_unlock(lock: &disk->open_mutex);
1603	out:
1604	ublk_put_disk(disk);
1605	}
1606
1607	/*
1608	* Use this function to ensure that ->canceling is consistently set for
1609	* the device and all queues. Do not set these flags directly.
1610	*
1611	* Caller must ensure that:
1612	* - cancel_mutex is held. This ensures that there is no concurrent
1613	* access to ub->canceling and no concurrent writes to ubq->canceling.
1614	* - there are no concurrent reads of ubq->canceling from the queue_rq
1615	* path. This can be done by quiescing the queue, or through other
1616	* means.
1617	*/
1618	static void ublk_set_canceling(struct ublk_device *ub, bool canceling)
1619	__must_hold(&ub->cancel_mutex)
1620	{
1621	int i;
1622
1623	ub->canceling = canceling;
1624	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1625	ublk_get_queue(dev: ub, qid: i)->canceling = canceling;
1626	}
1627
1628	static bool ublk_check_and_reset_active_ref(struct ublk_device *ub)
1629	{
1630	int i, j;
1631
1632	if (!ublk_dev_need_req_ref(ub))
1633	return false;
1634
1635	for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
1636	struct ublk_queue *ubq = ublk_get_queue(dev: ub, qid: i);
1637
1638	for (j = 0; j < ubq->q_depth; j++) {
1639	struct ublk_io *io = &ubq->ios[j];
1640	unsigned int refs = refcount_read(r: &io->ref) +
1641	io->task_registered_buffers;
1642
1643	/*
1644	* UBLK_REFCOUNT_INIT or zero means no active
1645	* reference
1646	*/
1647	if (refs != UBLK_REFCOUNT_INIT && refs != 0)
1648	return true;
1649
1650	/* reset to zero if the io hasn't active references */
1651	refcount_set(r: &io->ref, n: 0);
1652	io->task_registered_buffers = 0;
1653	}
1654	}
1655	return false;
1656	}
1657
1658	static void ublk_ch_release_work_fn(struct work_struct *work)
1659	{
1660	struct ublk_device *ub =
1661	container_of(work, struct ublk_device, exit_work.work);
1662	struct gendisk *disk;
1663	int i;
1664
1665	/*
1666	* For zero-copy and auto buffer register modes, I/O references
1667	* might not be dropped naturally when the daemon is killed, but
1668	* io_uring guarantees that registered bvec kernel buffers are
1669	* unregistered finally when freeing io_uring context, then the
1670	* active references are dropped.
1671	*
1672	* Wait until active references are dropped for avoiding use-after-free
1673	*
1674	* registered buffer may be unregistered in io_ring's release hander,
1675	* so have to wait by scheduling work function for avoiding the two
1676	* file release dependency.
1677	*/
1678	if (ublk_check_and_reset_active_ref(ub)) {
1679	schedule_delayed_work(dwork: &ub->exit_work, delay: 1);
1680	return;
1681	}
1682
1683	/*
1684	* disk isn't attached yet, either device isn't live, or it has
1685	* been removed already, so we needn't to do anything
1686	*/
1687	disk = ublk_get_disk(ub);
1688	if (!disk)
1689	goto out;
1690
1691	/*
1692	* All uring_cmd are done now, so abort any request outstanding to
1693	* the ublk server
1694	*
1695	* This can be done in lockless way because ublk server has been
1696	* gone
1697	*
1698	* More importantly, we have to provide forward progress guarantee
1699	* without holding ub->mutex, otherwise control task grabbing
1700	* ub->mutex triggers deadlock
1701	*
1702	* All requests may be inflight, so ->canceling may not be set, set
1703	* it now.
1704	*/
1705	mutex_lock(&ub->cancel_mutex);
1706	ublk_set_canceling(ub, canceling: true);
1707	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1708	ublk_abort_queue(ub, ubq: ublk_get_queue(dev: ub, qid: i));
1709	mutex_unlock(lock: &ub->cancel_mutex);
1710	blk_mq_kick_requeue_list(q: disk->queue);
1711
1712	/*
1713	* All infligh requests have been completed or requeued and any new
1714	* request will be failed or requeued via `->canceling` now, so it is
1715	* fine to grab ub->mutex now.
1716	*/
1717	mutex_lock(&ub->mutex);
1718
1719	/* double check after grabbing lock */
1720	if (!ub->ub_disk)
1721	goto unlock;
1722
1723	/*
1724	* Transition the device to the nosrv state. What exactly this
1725	* means depends on the recovery flags
1726	*/
1727	if (ublk_nosrv_should_stop_dev(ub)) {
1728	/*
1729	* Allow any pending/future I/O to pass through quickly
1730	* with an error. This is needed because del_gendisk
1731	* waits for all pending I/O to complete
1732	*/
1733	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1734	WRITE_ONCE(ublk_get_queue(ub, i)->force_abort, true);
1735
1736	ublk_stop_dev_unlocked(ub);
1737	} else {
1738	if (ublk_nosrv_dev_should_queue_io(ub)) {
1739	/* ->canceling is set and all requests are aborted */
1740	ub->dev_info.state = UBLK_S_DEV_QUIESCED;
1741	} else {
1742	ub->dev_info.state = UBLK_S_DEV_FAIL_IO;
1743	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1744	WRITE_ONCE(ublk_get_queue(ub, i)->fail_io, true);
1745	}
1746	}
1747	unlock:
1748	mutex_unlock(lock: &ub->mutex);
1749	ublk_put_disk(disk);
1750
1751	/* all uring_cmd has been done now, reset device & ubq */
1752	ublk_reset_ch_dev(ub);
1753	out:
1754	clear_bit(UB_STATE_OPEN, addr: &ub->state);
1755
1756	/* put the reference grabbed in ublk_ch_release() */
1757	ublk_put_device(ub);
1758	}
1759
1760	static int ublk_ch_release(struct inode *inode, struct file *filp)
1761	{
1762	struct ublk_device *ub = filp->private_data;
1763
1764	/*
1765	* Grab ublk device reference, so it won't be gone until we are
1766	* really released from work function.
1767	*/
1768	ublk_get_device(ub);
1769
1770	INIT_DELAYED_WORK(&ub->exit_work, ublk_ch_release_work_fn);
1771	schedule_delayed_work(dwork: &ub->exit_work, delay: 0);
1772	return 0;
1773	}
1774
1775	/* map pre-allocated per-queue cmd buffer to ublksrv daemon */
1776	static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma)
1777	{
1778	struct ublk_device *ub = filp->private_data;
1779	size_t sz = vma->vm_end - vma->vm_start;
1780	unsigned max_sz = ublk_max_cmd_buf_size();
1781	unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT;
1782	int q_id, ret = 0;
1783
1784	spin_lock(lock: &ub->lock);
1785	if (!ub->mm)
1786	ub->mm = current->mm;
1787	if (current->mm != ub->mm)
1788	ret = -EINVAL;
1789	spin_unlock(lock: &ub->lock);
1790
1791	if (ret)
1792	return ret;
1793
1794	if (vma->vm_flags & VM_WRITE)
1795	return -EPERM;
1796
1797	end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz;
1798	if (phys_off < UBLKSRV_CMD_BUF_OFFSET || phys_off >= end)
1799	return -EINVAL;
1800
1801	q_id = (phys_off - UBLKSRV_CMD_BUF_OFFSET) / max_sz;
1802	pr_devel("%s: qid %d, pid %d, addr %lx pg_off %lx sz %lu\n",
1803	__func__, q_id, current->pid, vma->vm_start,
1804	phys_off, (unsigned long)sz);
1805
1806	if (sz != ublk_queue_cmd_buf_size(ub))
1807	return -EINVAL;
1808
1809	pfn = virt_to_phys(address: ublk_queue_cmd_buf(ub, q_id)) >> PAGE_SHIFT;
1810	return remap_pfn_range(vma, addr: vma->vm_start, pfn, size: sz, pgprot: vma->vm_page_prot);
1811	}
1812
1813	static void __ublk_fail_req(struct ublk_device *ub, struct ublk_io *io,
1814	struct request *req)
1815	{
1816	WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_ACTIVE);
1817
1818	if (ublk_nosrv_should_reissue_outstanding(ub))
1819	blk_mq_requeue_request(rq: req, kick_requeue_list: false);
1820	else {
1821	io->res = -EIO;
1822	__ublk_complete_rq(req, io, need_map: ublk_dev_need_map_io(ub));
1823	}
1824	}
1825
1826	/*
1827	* Called from ublk char device release handler, when any uring_cmd is
1828	* done, meantime request queue is "quiesced" since all inflight requests
1829	* can't be completed because ublk server is dead.
1830	*
1831	* So no one can hold our request IO reference any more, simply ignore the
1832	* reference, and complete the request immediately
1833	*/
1834	static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq)
1835	{
1836	int i;
1837
1838	for (i = 0; i < ubq->q_depth; i++) {
1839	struct ublk_io *io = &ubq->ios[i];
1840
1841	if (io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)
1842	__ublk_fail_req(ub, io, req: io->req);
1843	}
1844	}
1845
1846	static void ublk_start_cancel(struct ublk_device *ub)
1847	{
1848	struct gendisk *disk = ublk_get_disk(ub);
1849
1850	/* Our disk has been dead */
1851	if (!disk)
1852	return;
1853
1854	mutex_lock(&ub->cancel_mutex);
1855	if (ub->canceling)
1856	goto out;
1857	/*
1858	* Now we are serialized with ublk_queue_rq()
1859	*
1860	* Make sure that ubq->canceling is set when queue is frozen,
1861	* because ublk_queue_rq() has to rely on this flag for avoiding to
1862	* touch completed uring_cmd
1863	*/
1864	blk_mq_quiesce_queue(q: disk->queue);
1865	ublk_set_canceling(ub, canceling: true);
1866	blk_mq_unquiesce_queue(q: disk->queue);
1867	out:
1868	mutex_unlock(lock: &ub->cancel_mutex);
1869	ublk_put_disk(disk);
1870	}
1871
1872	static void ublk_cancel_cmd(struct ublk_queue *ubq, unsigned tag,
1873	unsigned int issue_flags)
1874	{
1875	struct ublk_io *io = &ubq->ios[tag];
1876	struct ublk_device *ub = ubq->dev;
1877	struct request *req;
1878	bool done;
1879
1880	if (!(io->flags & UBLK_IO_FLAG_ACTIVE))
1881	return;
1882
1883	/*
1884	* Don't try to cancel this command if the request is started for
1885	* avoiding race between io_uring_cmd_done() and
1886	* io_uring_cmd_complete_in_task().
1887	*
1888	* Either the started request will be aborted via __ublk_abort_rq(),
1889	* then this uring_cmd is canceled next time, or it will be done in
1890	* task work function ublk_dispatch_req() because io_uring guarantees
1891	* that ublk_dispatch_req() is always called
1892	*/
1893	req = blk_mq_tag_to_rq(tags: ub->tag_set.tags[ubq->q_id], tag);
1894	if (req && blk_mq_request_started(rq: req) && req->tag == tag)
1895	return;
1896
1897	spin_lock(lock: &ubq->cancel_lock);
1898	done = !!(io->flags & UBLK_IO_FLAG_CANCELED);
1899	if (!done)
1900	io->flags |= UBLK_IO_FLAG_CANCELED;
1901	spin_unlock(lock: &ubq->cancel_lock);
1902
1903	if (!done)
1904	io_uring_cmd_done(ioucmd: io->cmd, UBLK_IO_RES_ABORT, issue_flags);
1905	}
1906
1907	/*
1908	* The ublk char device won't be closed when calling cancel fn, so both
1909	* ublk device and queue are guaranteed to be live
1910	*
1911	* Two-stage cancel:
1912	*
1913	* - make every active uring_cmd done in ->cancel_fn()
1914	*
1915	* - aborting inflight ublk IO requests in ublk char device release handler,
1916	* which depends on 1st stage because device can only be closed iff all
1917	* uring_cmd are done
1918	*
1919	* Do _not_ try to acquire ub->mutex before all inflight requests are
1920	* aborted, otherwise deadlock may be caused.
1921	*/
1922	static void ublk_uring_cmd_cancel_fn(struct io_uring_cmd *cmd,
1923	unsigned int issue_flags)
1924	{
1925	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(ioucmd: cmd);
1926	struct ublk_queue *ubq = pdu->ubq;
1927	struct task_struct *task;
1928	struct ublk_io *io;
1929
1930	if (WARN_ON_ONCE(!ubq))
1931	return;
1932
1933	if (WARN_ON_ONCE(pdu->tag >= ubq->q_depth))
1934	return;
1935
1936	task = io_uring_cmd_get_task(cmd);
1937	io = &ubq->ios[pdu->tag];
1938	if (WARN_ON_ONCE(task && task != io->task))
1939	return;
1940
1941	ublk_start_cancel(ub: ubq->dev);
1942
1943	WARN_ON_ONCE(io->cmd != cmd);
1944	ublk_cancel_cmd(ubq, tag: pdu->tag, issue_flags);
1945	}
1946
1947	static inline bool ublk_dev_ready(const struct ublk_device *ub)
1948	{
1949	u32 total = (u32)ub->dev_info.nr_hw_queues * ub->dev_info.queue_depth;
1950
1951	return ub->nr_io_ready == total;
1952	}
1953
1954	static void ublk_cancel_queue(struct ublk_queue *ubq)
1955	{
1956	int i;
1957
1958	for (i = 0; i < ubq->q_depth; i++)
1959	ublk_cancel_cmd(ubq, tag: i, issue_flags: IO_URING_F_UNLOCKED);
1960	}
1961
1962	/* Cancel all pending commands, must be called after del_gendisk() returns */
1963	static void ublk_cancel_dev(struct ublk_device *ub)
1964	{
1965	int i;
1966
1967	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
1968	ublk_cancel_queue(ubq: ublk_get_queue(dev: ub, qid: i));
1969	}
1970
1971	static bool ublk_check_inflight_rq(struct request *rq, void *data)
1972	{
1973	bool *idle = data;
1974
1975	if (blk_mq_request_started(rq)) {
1976	*idle = false;
1977	return false;
1978	}
1979	return true;
1980	}
1981
1982	static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub)
1983	{
1984	bool idle;
1985
1986	WARN_ON_ONCE(!blk_queue_quiesced(ub->ub_disk->queue));
1987	while (true) {
1988	idle = true;
1989	blk_mq_tagset_busy_iter(tagset: &ub->tag_set,
1990	fn: ublk_check_inflight_rq, priv: &idle);
1991	if (idle)
1992	break;
1993	msleep(UBLK_REQUEUE_DELAY_MS);
1994	}
1995	}
1996
1997	static void ublk_force_abort_dev(struct ublk_device *ub)
1998	{
1999	int i;
2000
2001	pr_devel("%s: force abort ub: dev_id %d state %s\n",
2002	__func__, ub->dev_info.dev_id,
2003	ub->dev_info.state == UBLK_S_DEV_LIVE ?
2004	"LIVE" : "QUIESCED");
2005	blk_mq_quiesce_queue(q: ub->ub_disk->queue);
2006	if (ub->dev_info.state == UBLK_S_DEV_LIVE)
2007	ublk_wait_tagset_rqs_idle(ub);
2008
2009	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
2010	ublk_get_queue(dev: ub, qid: i)->force_abort = true;
2011	blk_mq_unquiesce_queue(q: ub->ub_disk->queue);
2012	/* We may have requeued some rqs in ublk_quiesce_queue() */
2013	blk_mq_kick_requeue_list(q: ub->ub_disk->queue);
2014	}
2015
2016	static struct gendisk *ublk_detach_disk(struct ublk_device *ub)
2017	{
2018	struct gendisk *disk;
2019
2020	/* Sync with ublk_abort_queue() by holding the lock */
2021	spin_lock(lock: &ub->lock);
2022	disk = ub->ub_disk;
2023	ub->dev_info.state = UBLK_S_DEV_DEAD;
2024	ub->dev_info.ublksrv_pid = -1;
2025	ub->ub_disk = NULL;
2026	spin_unlock(lock: &ub->lock);
2027
2028	return disk;
2029	}
2030
2031	static void ublk_stop_dev_unlocked(struct ublk_device *ub)
2032	__must_hold(&ub->mutex)
2033	{
2034	struct gendisk *disk;
2035
2036	if (ub->dev_info.state == UBLK_S_DEV_DEAD)
2037	return;
2038
2039	if (ublk_nosrv_dev_should_queue_io(ub))
2040	ublk_force_abort_dev(ub);
2041	del_gendisk(gp: ub->ub_disk);
2042	disk = ublk_detach_disk(ub);
2043	put_disk(disk);
2044	}
2045
2046	static void ublk_stop_dev(struct ublk_device *ub)
2047	{
2048	mutex_lock(&ub->mutex);
2049	ublk_stop_dev_unlocked(ub);
2050	mutex_unlock(lock: &ub->mutex);
2051	cancel_work_sync(work: &ub->partition_scan_work);
2052	ublk_cancel_dev(ub);
2053	}
2054
2055	/* reset ublk io_uring queue & io flags */
2056	static void ublk_reset_io_flags(struct ublk_device *ub)
2057	{
2058	int i, j;
2059
2060	for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
2061	struct ublk_queue *ubq = ublk_get_queue(dev: ub, qid: i);
2062
2063	/* UBLK_IO_FLAG_CANCELED can be cleared now */
2064	spin_lock(lock: &ubq->cancel_lock);
2065	for (j = 0; j < ubq->q_depth; j++)
2066	ubq->ios[j].flags &= ~UBLK_IO_FLAG_CANCELED;
2067	spin_unlock(lock: &ubq->cancel_lock);
2068	ubq->fail_io = false;
2069	}
2070	mutex_lock(&ub->cancel_mutex);
2071	ublk_set_canceling(ub, canceling: false);
2072	mutex_unlock(lock: &ub->cancel_mutex);
2073	}
2074
2075	/* device can only be started after all IOs are ready */
2076	static void ublk_mark_io_ready(struct ublk_device *ub)
2077	__must_hold(&ub->mutex)
2078	{
2079	if (!ub->unprivileged_daemons && !capable(CAP_SYS_ADMIN))
2080	ub->unprivileged_daemons = true;
2081
2082	ub->nr_io_ready++;
2083	if (ublk_dev_ready(ub)) {
2084	/* now we are ready for handling ublk io request */
2085	ublk_reset_io_flags(ub);
2086	complete_all(&ub->completion);
2087	}
2088	}
2089
2090	static inline int ublk_check_cmd_op(u32 cmd_op)
2091	{
2092	u32 ioc_type = _IOC_TYPE(cmd_op);
2093
2094	if (!IS_ENABLED(CONFIG_BLKDEV_UBLK_LEGACY_OPCODES) && ioc_type != 'u')
2095	return -EOPNOTSUPP;
2096
2097	if (ioc_type != 'u' && ioc_type != 0)
2098	return -EOPNOTSUPP;
2099
2100	return 0;
2101	}
2102
2103	static inline int ublk_set_auto_buf_reg(struct ublk_io *io, struct io_uring_cmd *cmd)
2104	{
2105	struct ublk_auto_buf_reg buf;
2106
2107	buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr));
2108
2109	if (buf.reserved0 || buf.reserved1)
2110	return -EINVAL;
2111
2112	if (buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK)
2113	return -EINVAL;
2114	io->buf.auto_reg = buf;
2115	return 0;
2116	}
2117
2118	static int ublk_handle_auto_buf_reg(struct ublk_io *io,
2119	struct io_uring_cmd *cmd,
2120	u16 *buf_idx)
2121	{
2122	if (io->flags & UBLK_IO_FLAG_AUTO_BUF_REG) {
2123	io->flags &= ~UBLK_IO_FLAG_AUTO_BUF_REG;
2124
2125	/*
2126	* `UBLK_F_AUTO_BUF_REG` only works iff `UBLK_IO_FETCH_REQ`
2127	* and `UBLK_IO_COMMIT_AND_FETCH_REQ` are issued from same
2128	* `io_ring_ctx`.
2129	*
2130	* If this uring_cmd's io_ring_ctx isn't same with the
2131	* one for registering the buffer, it is ublk server's
2132	* responsibility for unregistering the buffer, otherwise
2133	* this ublk request gets stuck.
2134	*/
2135	if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd))
2136	*buf_idx = io->buf.auto_reg.index;
2137	}
2138
2139	return ublk_set_auto_buf_reg(io, cmd);
2140	}
2141
2142	/* Once we return, `io->req` can't be used any more */
2143	static inline struct request *
2144	ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd)
2145	{
2146	struct request *req = io->req;
2147
2148	io->cmd = cmd;
2149	io->flags |= UBLK_IO_FLAG_ACTIVE;
2150	/* now this cmd slot is owned by ublk driver */
2151	io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV;
2152
2153	return req;
2154	}
2155
2156	static inline int
2157	ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io,
2158	struct io_uring_cmd *cmd, unsigned long buf_addr,
2159	u16 *buf_idx)
2160	{
2161	if (ublk_dev_support_auto_buf_reg(ub))
2162	return ublk_handle_auto_buf_reg(io, cmd, buf_idx);
2163
2164	io->buf.addr = buf_addr;
2165	return 0;
2166	}
2167
2168	static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,
2169	unsigned int issue_flags,
2170	struct ublk_queue *ubq, unsigned int tag)
2171	{
2172	struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(ioucmd: cmd);
2173
2174	/*
2175	* Safe to refer to @ubq since ublk_queue won't be died until its
2176	* commands are completed
2177	*/
2178	pdu->ubq = ubq;
2179	pdu->tag = tag;
2180	io_uring_cmd_mark_cancelable(cmd, issue_flags);
2181	}
2182
2183	static void ublk_io_release(void *priv)
2184	{
2185	struct request *rq = priv;
2186	struct ublk_queue *ubq = rq->mq_hctx->driver_data;
2187	struct ublk_io *io = &ubq->ios[rq->tag];
2188
2189	/*
2190	* task_registered_buffers may be 0 if buffers were registered off task
2191	* but unregistered on task. Or after UBLK_IO_COMMIT_AND_FETCH_REQ.
2192	*/
2193	if (current == io->task && io->task_registered_buffers)
2194	io->task_registered_buffers--;
2195	else
2196	ublk_put_req_ref(io, req: rq);
2197	}
2198
2199	static int ublk_register_io_buf(struct io_uring_cmd *cmd,
2200	struct ublk_device *ub,
2201	u16 q_id, u16 tag,
2202	struct ublk_io *io,
2203	unsigned int index, unsigned int issue_flags)
2204	{
2205	struct request *req;
2206	int ret;
2207
2208	if (!ublk_dev_support_zero_copy(ub))
2209	return -EINVAL;
2210
2211	req = __ublk_check_and_get_req(ub, q_id, tag, io, offset: 0);
2212	if (!req)
2213	return -EINVAL;
2214
2215	ret = io_buffer_register_bvec(cmd, rq: req, release: ublk_io_release, index,
2216	issue_flags);
2217	if (ret) {
2218	ublk_put_req_ref(io, req);
2219	return ret;
2220	}
2221
2222	return 0;
2223	}
2224
2225	static int
2226	ublk_daemon_register_io_buf(struct io_uring_cmd *cmd,
2227	struct ublk_device *ub,
2228	u16 q_id, u16 tag, struct ublk_io *io,
2229	unsigned index, unsigned issue_flags)
2230	{
2231	unsigned new_registered_buffers;
2232	struct request *req = io->req;
2233	int ret;
2234
2235	/*
2236	* Ensure there are still references for ublk_sub_req_ref() to release.
2237	* If not, fall back on the thread-safe buffer registration.
2238	*/
2239	new_registered_buffers = io->task_registered_buffers + 1;
2240	if (unlikely(new_registered_buffers >= UBLK_REFCOUNT_INIT))
2241	return ublk_register_io_buf(cmd, ub, q_id, tag, io, index,
2242	issue_flags);
2243
2244	if (!ublk_dev_support_zero_copy(ub) || !ublk_rq_has_data(rq: req))
2245	return -EINVAL;
2246
2247	ret = io_buffer_register_bvec(cmd, rq: req, release: ublk_io_release, index,
2248	issue_flags);
2249	if (ret)
2250	return ret;
2251
2252	io->task_registered_buffers = new_registered_buffers;
2253	return 0;
2254	}
2255
2256	static int ublk_unregister_io_buf(struct io_uring_cmd *cmd,
2257	const struct ublk_device *ub,
2258	unsigned int index, unsigned int issue_flags)
2259	{
2260	if (!(ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY))
2261	return -EINVAL;
2262
2263	return io_buffer_unregister_bvec(cmd, index, issue_flags);
2264	}
2265
2266	static int ublk_check_fetch_buf(const struct ublk_device *ub, __u64 buf_addr)
2267	{
2268	if (ublk_dev_need_map_io(ub)) {
2269	/*
2270	* FETCH_RQ has to provide IO buffer if NEED GET
2271	* DATA is not enabled
2272	*/
2273	if (!buf_addr && !ublk_dev_need_get_data(ub))
2274	return -EINVAL;
2275	} else if (buf_addr) {
2276	/* User copy requires addr to be unset */
2277	return -EINVAL;
2278	}
2279	return 0;
2280	}
2281
2282	static int __ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub,
2283	struct ublk_io *io)
2284	{
2285	/* UBLK_IO_FETCH_REQ is only allowed before dev is setup */
2286	if (ublk_dev_ready(ub))
2287	return -EBUSY;
2288
2289	/* allow each command to be FETCHed at most once */
2290	if (io->flags & UBLK_IO_FLAG_ACTIVE)
2291	return -EINVAL;
2292
2293	WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV);
2294
2295	ublk_fill_io_cmd(io, cmd);
2296
2297	WRITE_ONCE(io->task, get_task_struct(current));
2298	ublk_mark_io_ready(ub);
2299
2300	return 0;
2301	}
2302
2303	static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub,
2304	struct ublk_io *io, __u64 buf_addr)
2305	{
2306	int ret;
2307
2308	/*
2309	* When handling FETCH command for setting up ublk uring queue,
2310	* ub->mutex is the innermost lock, and we won't block for handling
2311	* FETCH, so it is fine even for IO_URING_F_NONBLOCK.
2312	*/
2313	mutex_lock(&ub->mutex);
2314	ret = __ublk_fetch(cmd, ub, io);
2315	if (!ret)
2316	ret = ublk_config_io_buf(ub, io, cmd, buf_addr, NULL);
2317	mutex_unlock(lock: &ub->mutex);
2318	return ret;
2319	}
2320
2321	static int ublk_check_commit_and_fetch(const struct ublk_device *ub,
2322	struct ublk_io *io, __u64 buf_addr)
2323	{
2324	struct request *req = io->req;
2325
2326	if (ublk_dev_need_map_io(ub)) {
2327	/*
2328	* COMMIT_AND_FETCH_REQ has to provide IO buffer if
2329	* NEED GET DATA is not enabled or it is Read IO.
2330	*/
2331	if (!buf_addr && (!ublk_dev_need_get_data(ub) ||
2332	req_op(req) == REQ_OP_READ))
2333	return -EINVAL;
2334	} else if (req_op(req) != REQ_OP_ZONE_APPEND && buf_addr) {
2335	/*
2336	* User copy requires addr to be unset when command is
2337	* not zone append
2338	*/
2339	return -EINVAL;
2340	}
2341
2342	return 0;
2343	}
2344
2345	static bool ublk_need_complete_req(const struct ublk_device *ub,
2346	struct ublk_io *io)
2347	{
2348	if (ublk_dev_need_req_ref(ub))
2349	return ublk_sub_req_ref(io);
2350	return true;
2351	}
2352
2353	static bool ublk_get_data(const struct ublk_queue *ubq, struct ublk_io *io,
2354	struct request *req)
2355	{
2356	/*
2357	* We have handled UBLK_IO_NEED_GET_DATA command,
2358	* so clear UBLK_IO_FLAG_NEED_GET_DATA now and just
2359	* do the copy work.
2360	*/
2361	io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA;
2362	/* update iod->addr because ublksrv may have passed a new io buffer */
2363	ublk_get_iod(ubq, tag: req->tag)->addr = io->buf.addr;
2364	pr_devel("%s: update iod->addr: qid %d tag %d io_flags %x addr %llx\n",
2365	__func__, ubq->q_id, req->tag, io->flags,
2366	ublk_get_iod(ubq, req->tag)->addr);
2367
2368	return ublk_start_io(ubq, req, io);
2369	}
2370
2371	static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd,
2372	unsigned int issue_flags)
2373	{
2374	/* May point to userspace-mapped memory */
2375	const struct ublksrv_io_cmd *ub_src = io_uring_sqe_cmd(sqe: cmd->sqe);
2376	u16 buf_idx = UBLK_INVALID_BUF_IDX;
2377	struct ublk_device *ub = cmd->file->private_data;
2378	struct ublk_queue *ubq;
2379	struct ublk_io *io = NULL;
2380	u32 cmd_op = cmd->cmd_op;
2381	u16 q_id = READ_ONCE(ub_src->q_id);
2382	u16 tag = READ_ONCE(ub_src->tag);
2383	s32 result = READ_ONCE(ub_src->result);
2384	u64 addr = READ_ONCE(ub_src->addr); /* unioned with zone_append_lba */
2385	struct request *req;
2386	int ret;
2387	bool compl;
2388
2389	WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED);
2390
2391	pr_devel("%s: received: cmd op %d queue %d tag %d result %d\n",
2392	__func__, cmd->cmd_op, q_id, tag, result);
2393
2394	ret = ublk_check_cmd_op(cmd_op);
2395	if (ret)
2396	goto out;
2397
2398	/*
2399	* io_buffer_unregister_bvec() doesn't access the ubq or io,
2400	* so no need to validate the q_id, tag, or task
2401	*/
2402	if (_IOC_NR(cmd_op) == UBLK_IO_UNREGISTER_IO_BUF)
2403	return ublk_unregister_io_buf(cmd, ub, index: addr, issue_flags);
2404
2405	ret = -EINVAL;
2406	if (q_id >= ub->dev_info.nr_hw_queues)
2407	goto out;
2408
2409	ubq = ublk_get_queue(dev: ub, qid: q_id);
2410
2411	if (tag >= ub->dev_info.queue_depth)
2412	goto out;
2413
2414	io = &ubq->ios[tag];
2415	/* UBLK_IO_FETCH_REQ can be handled on any task, which sets io->task */
2416	if (unlikely(_IOC_NR(cmd_op) == UBLK_IO_FETCH_REQ)) {
2417	ret = ublk_check_fetch_buf(ub, buf_addr: addr);
2418	if (ret)
2419	goto out;
2420	ret = ublk_fetch(cmd, ub, io, buf_addr: addr);
2421	if (ret)
2422	goto out;
2423
2424	ublk_prep_cancel(cmd, issue_flags, ubq, tag);
2425	return -EIOCBQUEUED;
2426	}
2427
2428	if (READ_ONCE(io->task) != current) {
2429	/*
2430	* ublk_register_io_buf() accesses only the io's refcount,
2431	* so can be handled on any task
2432	*/
2433	if (_IOC_NR(cmd_op) == UBLK_IO_REGISTER_IO_BUF)
2434	return ublk_register_io_buf(cmd, ub, q_id, tag, io,
2435	index: addr, issue_flags);
2436
2437	goto out;
2438	}
2439
2440	/* there is pending io cmd, something must be wrong */
2441	if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) {
2442	ret = -EBUSY;
2443	goto out;
2444	}
2445
2446	/*
2447	* ensure that the user issues UBLK_IO_NEED_GET_DATA
2448	* iff the driver have set the UBLK_IO_FLAG_NEED_GET_DATA.
2449	*/
2450	if ((!!(io->flags & UBLK_IO_FLAG_NEED_GET_DATA))
2451	^ (_IOC_NR(cmd_op) == UBLK_IO_NEED_GET_DATA))
2452	goto out;
2453
2454	switch (_IOC_NR(cmd_op)) {
2455	case UBLK_IO_REGISTER_IO_BUF:
2456	return ublk_daemon_register_io_buf(cmd, ub, q_id, tag, io, index: addr,
2457	issue_flags);
2458	case UBLK_IO_COMMIT_AND_FETCH_REQ:
2459	ret = ublk_check_commit_and_fetch(ub, io, buf_addr: addr);
2460	if (ret)
2461	goto out;
2462	io->res = result;
2463	req = ublk_fill_io_cmd(io, cmd);
2464	ret = ublk_config_io_buf(ub, io, cmd, buf_addr: addr, buf_idx: &buf_idx);
2465	compl = ublk_need_complete_req(ub, io);
2466
2467	/* can't touch 'ublk_io' any more */
2468	if (buf_idx != UBLK_INVALID_BUF_IDX)
2469	io_buffer_unregister_bvec(cmd, index: buf_idx, issue_flags);
2470	if (req_op(req) == REQ_OP_ZONE_APPEND)
2471	req->__sector = addr;
2472	if (compl)
2473	__ublk_complete_rq(req, io, need_map: ublk_dev_need_map_io(ub));
2474
2475	if (ret)
2476	goto out;
2477	break;
2478	case UBLK_IO_NEED_GET_DATA:
2479	/*
2480	* ublk_get_data() may fail and fallback to requeue, so keep
2481	* uring_cmd active first and prepare for handling new requeued
2482	* request
2483	*/
2484	req = ublk_fill_io_cmd(io, cmd);
2485	ret = ublk_config_io_buf(ub, io, cmd, buf_addr: addr, NULL);
2486	WARN_ON_ONCE(ret);
2487	if (likely(ublk_get_data(ubq, io, req))) {
2488	__ublk_prep_compl_io_cmd(io, req);
2489	return UBLK_IO_RES_OK;
2490	}
2491	break;
2492	default:
2493	goto out;
2494	}
2495	ublk_prep_cancel(cmd, issue_flags, ubq, tag);
2496	return -EIOCBQUEUED;
2497
2498	out:
2499	pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n",
2500	__func__, cmd_op, tag, ret, io ? io->flags : 0);
2501	return ret;
2502	}
2503
2504	static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,
2505	u16 q_id, u16 tag, struct ublk_io *io, size_t offset)
2506	{
2507	struct request *req;
2508
2509	/*
2510	* can't use io->req in case of concurrent UBLK_IO_COMMIT_AND_FETCH_REQ,
2511	* which would overwrite it with io->cmd
2512	*/
2513	req = blk_mq_tag_to_rq(tags: ub->tag_set.tags[q_id], tag);
2514	if (!req)
2515	return NULL;
2516
2517	if (!ublk_get_req_ref(io))
2518	return NULL;
2519
2520	if (unlikely(!blk_mq_request_started(req) || req->tag != tag))
2521	goto fail_put;
2522
2523	if (!ublk_rq_has_data(rq: req))
2524	goto fail_put;
2525
2526	if (offset > blk_rq_bytes(rq: req))
2527	goto fail_put;
2528
2529	return req;
2530	fail_put:
2531	ublk_put_req_ref(io, req);
2532	return NULL;
2533	}
2534
2535	static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)
2536	{
2537	unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
2538	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
2539	int ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
2540
2541	if (ret != -EIOCBQUEUED)
2542	io_uring_cmd_done(ioucmd: cmd, ret, issue_flags);
2543	}
2544
2545	static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
2546	{
2547	if (unlikely(issue_flags & IO_URING_F_CANCEL)) {
2548	ublk_uring_cmd_cancel_fn(cmd, issue_flags);
2549	return 0;
2550	}
2551
2552	/* well-implemented server won't run into unlocked */
2553	if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
2554	io_uring_cmd_complete_in_task(ioucmd: cmd, task_work_cb: ublk_ch_uring_cmd_cb);
2555	return -EIOCBQUEUED;
2556	}
2557
2558	return ublk_ch_uring_cmd_local(cmd, issue_flags);
2559	}
2560
2561	static inline bool ublk_check_ubuf_dir(const struct request *req,
2562	int ubuf_dir)
2563	{
2564	/* copy ubuf to request pages */
2565	if ((req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_DRV_IN) &&
2566	ubuf_dir == ITER_SOURCE)
2567	return true;
2568
2569	/* copy request pages to ubuf */
2570	if ((req_op(req) == REQ_OP_WRITE ||
2571	req_op(req) == REQ_OP_ZONE_APPEND) &&
2572	ubuf_dir == ITER_DEST)
2573	return true;
2574
2575	return false;
2576	}
2577
2578	static struct request *ublk_check_and_get_req(struct kiocb *iocb,
2579	struct iov_iter *iter, size_t *off, int dir,
2580	struct ublk_io **io)
2581	{
2582	struct ublk_device *ub = iocb->ki_filp->private_data;
2583	struct ublk_queue *ubq;
2584	struct request *req;
2585	size_t buf_off;
2586	u16 tag, q_id;
2587
2588	if (!user_backed_iter(i: iter))
2589	return ERR_PTR(error: -EACCES);
2590
2591	if (ub->dev_info.state == UBLK_S_DEV_DEAD)
2592	return ERR_PTR(error: -EACCES);
2593
2594	tag = ublk_pos_to_tag(pos: iocb->ki_pos);
2595	q_id = ublk_pos_to_hwq(pos: iocb->ki_pos);
2596	buf_off = ublk_pos_to_buf_off(pos: iocb->ki_pos);
2597
2598	if (q_id >= ub->dev_info.nr_hw_queues)
2599	return ERR_PTR(error: -EINVAL);
2600
2601	ubq = ublk_get_queue(dev: ub, qid: q_id);
2602	if (!ublk_dev_support_user_copy(ub))
2603	return ERR_PTR(error: -EACCES);
2604
2605	if (tag >= ub->dev_info.queue_depth)
2606	return ERR_PTR(error: -EINVAL);
2607
2608	*io = &ubq->ios[tag];
2609	req = __ublk_check_and_get_req(ub, q_id, tag, io: *io, offset: buf_off);
2610	if (!req)
2611	return ERR_PTR(error: -EINVAL);
2612
2613	if (!ublk_check_ubuf_dir(req, ubuf_dir: dir))
2614	goto fail;
2615
2616	*off = buf_off;
2617	return req;
2618	fail:
2619	ublk_put_req_ref(io: *io, req);
2620	return ERR_PTR(error: -EACCES);
2621	}
2622
2623	static ssize_t ublk_ch_read_iter(struct kiocb *iocb, struct iov_iter *to)
2624	{
2625	struct request *req;
2626	struct ublk_io *io;
2627	size_t buf_off;
2628	size_t ret;
2629
2630	req = ublk_check_and_get_req(iocb, iter: to, off: &buf_off, ITER_DEST, io: &io);
2631	if (IS_ERR(ptr: req))
2632	return PTR_ERR(ptr: req);
2633
2634	ret = ublk_copy_user_pages(req, offset: buf_off, uiter: to, ITER_DEST);
2635	ublk_put_req_ref(io, req);
2636
2637	return ret;
2638	}
2639
2640	static ssize_t ublk_ch_write_iter(struct kiocb *iocb, struct iov_iter *from)
2641	{
2642	struct request *req;
2643	struct ublk_io *io;
2644	size_t buf_off;
2645	size_t ret;
2646
2647	req = ublk_check_and_get_req(iocb, iter: from, off: &buf_off, ITER_SOURCE, io: &io);
2648	if (IS_ERR(ptr: req))
2649	return PTR_ERR(ptr: req);
2650
2651	ret = ublk_copy_user_pages(req, offset: buf_off, uiter: from, ITER_SOURCE);
2652	ublk_put_req_ref(io, req);
2653
2654	return ret;
2655	}
2656
2657	static const struct file_operations ublk_ch_fops = {
2658	.owner = THIS_MODULE,
2659	.open = ublk_ch_open,
2660	.release = ublk_ch_release,
2661	.read_iter = ublk_ch_read_iter,
2662	.write_iter = ublk_ch_write_iter,
2663	.uring_cmd = ublk_ch_uring_cmd,
2664	.mmap = ublk_ch_mmap,
2665	};
2666
2667	static void ublk_deinit_queue(struct ublk_device *ub, int q_id)
2668	{
2669	struct ublk_queue *ubq = ub->queues[q_id];
2670	int size, i;
2671
2672	if (!ubq)
2673	return;
2674
2675	size = ublk_queue_cmd_buf_size(ub);
2676
2677	for (i = 0; i < ubq->q_depth; i++) {
2678	struct ublk_io *io = &ubq->ios[i];
2679	if (io->task)
2680	put_task_struct(t: io->task);
2681	WARN_ON_ONCE(refcount_read(&io->ref));
2682	WARN_ON_ONCE(io->task_registered_buffers);
2683	}
2684
2685	if (ubq->io_cmd_buf)
2686	free_pages(addr: (unsigned long)ubq->io_cmd_buf, order: get_order(size));
2687
2688	kvfree(addr: ubq);
2689	ub->queues[q_id] = NULL;
2690	}
2691
2692	static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id)
2693	{
2694	unsigned int cpu;
2695
2696	/* Find first CPU mapped to this queue */
2697	for_each_possible_cpu(cpu) {
2698	if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[cpu] == q_id)
2699	return cpu_to_node(cpu);
2700	}
2701
2702	return NUMA_NO_NODE;
2703	}
2704
2705	static int ublk_init_queue(struct ublk_device *ub, int q_id)
2706	{
2707	int depth = ub->dev_info.queue_depth;
2708	gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO;
2709	struct ublk_queue *ubq;
2710	struct page *page;
2711	int numa_node;
2712	int size;
2713
2714	/* Determine NUMA node based on queue's CPU affinity */
2715	numa_node = ublk_get_queue_numa_node(ub, q_id);
2716
2717	/* Allocate queue structure on local NUMA node */
2718	ubq = kvzalloc_node(struct_size(ubq, ios, depth), GFP_KERNEL,
2719	numa_node);
2720	if (!ubq)
2721	return -ENOMEM;
2722
2723	spin_lock_init(&ubq->cancel_lock);
2724	ubq->flags = ub->dev_info.flags;
2725	ubq->q_id = q_id;
2726	ubq->q_depth = depth;
2727	size = ublk_queue_cmd_buf_size(ub);
2728
2729	/* Allocate I/O command buffer on local NUMA node */
2730	page = alloc_pages_node(numa_node, gfp_flags, get_order(size));
2731	if (!page) {
2732	kvfree(addr: ubq);
2733	return -ENOMEM;
2734	}
2735	ubq->io_cmd_buf = page_address(page);
2736
2737	ub->queues[q_id] = ubq;
2738	ubq->dev = ub;
2739	return 0;
2740	}
2741
2742	static void ublk_deinit_queues(struct ublk_device *ub)
2743	{
2744	int i;
2745
2746	for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
2747	ublk_deinit_queue(ub, q_id: i);
2748	}
2749
2750	static int ublk_init_queues(struct ublk_device *ub)
2751	{
2752	int i, ret;
2753
2754	for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
2755	ret = ublk_init_queue(ub, q_id: i);
2756	if (ret)
2757	goto fail;
2758	}
2759
2760	init_completion(x: &ub->completion);
2761	return 0;
2762
2763	fail:
2764	ublk_deinit_queues(ub);
2765	return ret;
2766	}
2767
2768	static int ublk_alloc_dev_number(struct ublk_device *ub, int idx)
2769	{
2770	int i = idx;
2771	int err;
2772
2773	spin_lock(lock: &ublk_idr_lock);
2774	/* allocate id, if @id >= 0, we're requesting that specific id */
2775	if (i >= 0) {
2776	err = idr_alloc(&ublk_index_idr, ptr: ub, start: i, end: i + 1, GFP_NOWAIT);
2777	if (err == -ENOSPC)
2778	err = -EEXIST;
2779	} else {
2780	err = idr_alloc(&ublk_index_idr, ptr: ub, start: 0, UBLK_MAX_UBLKS,
2781	GFP_NOWAIT);
2782	}
2783	spin_unlock(lock: &ublk_idr_lock);
2784
2785	if (err >= 0)
2786	ub->ub_number = err;
2787
2788	return err;
2789	}
2790
2791	static void ublk_free_dev_number(struct ublk_device *ub)
2792	{
2793	spin_lock(lock: &ublk_idr_lock);
2794	idr_remove(&ublk_index_idr, id: ub->ub_number);
2795	wake_up_all(&ublk_idr_wq);
2796	spin_unlock(lock: &ublk_idr_lock);
2797	}
2798
2799	static void ublk_cdev_rel(struct device *dev)
2800	{
2801	struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev);
2802
2803	blk_mq_free_tag_set(set: &ub->tag_set);
2804	ublk_deinit_queues(ub);
2805	ublk_free_dev_number(ub);
2806	mutex_destroy(lock: &ub->mutex);
2807	mutex_destroy(lock: &ub->cancel_mutex);
2808	kfree(objp: ub);
2809	}
2810
2811	static int ublk_add_chdev(struct ublk_device *ub)
2812	{
2813	struct device *dev = &ub->cdev_dev;
2814	int minor = ub->ub_number;
2815	int ret;
2816
2817	dev->parent = ublk_misc.this_device;
2818	dev->devt = MKDEV(MAJOR(ublk_chr_devt), minor);
2819	dev->class = &ublk_chr_class;
2820	dev->release = ublk_cdev_rel;
2821	device_initialize(dev);
2822
2823	ret = dev_set_name(dev, name: "ublkc%d", minor);
2824	if (ret)
2825	goto fail;
2826
2827	cdev_init(&ub->cdev, &ublk_ch_fops);
2828	ret = cdev_device_add(cdev: &ub->cdev, dev);
2829	if (ret)
2830	goto fail;
2831
2832	if (ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV)
2833	unprivileged_ublks_added++;
2834	return 0;
2835	fail:
2836	put_device(dev);
2837	return ret;
2838	}
2839
2840	/* align max io buffer size with PAGE_SIZE */
2841	static void ublk_align_max_io_size(struct ublk_device *ub)
2842	{
2843	unsigned int max_io_bytes = ub->dev_info.max_io_buf_bytes;
2844
2845	ub->dev_info.max_io_buf_bytes =
2846	round_down(max_io_bytes, PAGE_SIZE);
2847	}
2848
2849	static int ublk_add_tag_set(struct ublk_device *ub)
2850	{
2851	ub->tag_set.ops = &ublk_mq_ops;
2852	ub->tag_set.nr_hw_queues = ub->dev_info.nr_hw_queues;
2853	ub->tag_set.queue_depth = ub->dev_info.queue_depth;
2854	ub->tag_set.numa_node = NUMA_NO_NODE;
2855	ub->tag_set.driver_data = ub;
2856	return blk_mq_alloc_tag_set(set: &ub->tag_set);
2857	}
2858
2859	static void ublk_remove(struct ublk_device *ub)
2860	{
2861	bool unprivileged;
2862
2863	ublk_stop_dev(ub);
2864	cdev_device_del(cdev: &ub->cdev, dev: &ub->cdev_dev);
2865	unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV;
2866	ublk_put_device(ub);
2867
2868	if (unprivileged)
2869	unprivileged_ublks_added--;
2870	}
2871
2872	static struct ublk_device *ublk_get_device_from_id(int idx)
2873	{
2874	struct ublk_device *ub = NULL;
2875
2876	if (idx < 0)
2877	return NULL;
2878
2879	spin_lock(lock: &ublk_idr_lock);
2880	ub = idr_find(&ublk_index_idr, id: idx);
2881	if (ub)
2882	ub = ublk_get_device(ub);
2883	spin_unlock(lock: &ublk_idr_lock);
2884
2885	return ub;
2886	}
2887
2888	static bool ublk_validate_user_pid(struct ublk_device *ub, pid_t ublksrv_pid)
2889	{
2890	rcu_read_lock();
2891	ublksrv_pid = pid_nr(pid: find_vpid(nr: ublksrv_pid));
2892	rcu_read_unlock();
2893
2894	return ub->ublksrv_tgid == ublksrv_pid;
2895	}
2896
2897	static int ublk_ctrl_start_dev(struct ublk_device *ub,
2898	const struct ublksrv_ctrl_cmd *header)
2899	{
2900	const struct ublk_param_basic *p = &ub->params.basic;
2901	int ublksrv_pid = (int)header->data[0];
2902	struct queue_limits lim = {
2903	.logical_block_size = 1 << p->logical_bs_shift,
2904	.physical_block_size = 1 << p->physical_bs_shift,
2905	.io_min = 1 << p->io_min_shift,
2906	.io_opt = 1 << p->io_opt_shift,
2907	.max_hw_sectors = p->max_sectors,
2908	.chunk_sectors = p->chunk_sectors,
2909	.virt_boundary_mask = p->virt_boundary_mask,
2910	.max_segments = USHRT_MAX,
2911	.max_segment_size = UINT_MAX,
2912	.dma_alignment = 3,
2913	};
2914	struct gendisk *disk;
2915	int ret = -EINVAL;
2916
2917	if (ublksrv_pid <= 0)
2918	return -EINVAL;
2919	if (!(ub->params.types & UBLK_PARAM_TYPE_BASIC))
2920	return -EINVAL;
2921
2922	if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) {
2923	const struct ublk_param_discard *pd = &ub->params.discard;
2924
2925	lim.discard_alignment = pd->discard_alignment;
2926	lim.discard_granularity = pd->discard_granularity;
2927	lim.max_hw_discard_sectors = pd->max_discard_sectors;
2928	lim.max_write_zeroes_sectors = pd->max_write_zeroes_sectors;
2929	lim.max_discard_segments = pd->max_discard_segments;
2930	}
2931
2932	if (ub->params.types & UBLK_PARAM_TYPE_ZONED) {
2933	const struct ublk_param_zoned *p = &ub->params.zoned;
2934
2935	if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED))
2936	return -EOPNOTSUPP;
2937
2938	lim.features |= BLK_FEAT_ZONED;
2939	lim.max_active_zones = p->max_active_zones;
2940	lim.max_open_zones = p->max_open_zones;
2941	lim.max_hw_zone_append_sectors = p->max_zone_append_sectors;
2942	}
2943
2944	if (ub->params.basic.attrs & UBLK_ATTR_VOLATILE_CACHE) {
2945	lim.features |= BLK_FEAT_WRITE_CACHE;
2946	if (ub->params.basic.attrs & UBLK_ATTR_FUA)
2947	lim.features |= BLK_FEAT_FUA;
2948	}
2949
2950	if (ub->params.basic.attrs & UBLK_ATTR_ROTATIONAL)
2951	lim.features |= BLK_FEAT_ROTATIONAL;
2952
2953	if (ub->params.types & UBLK_PARAM_TYPE_DMA_ALIGN)
2954	lim.dma_alignment = ub->params.dma.alignment;
2955
2956	if (ub->params.types & UBLK_PARAM_TYPE_SEGMENT) {
2957	lim.seg_boundary_mask = ub->params.seg.seg_boundary_mask;
2958	lim.max_segment_size = ub->params.seg.max_segment_size;
2959	lim.max_segments = ub->params.seg.max_segments;
2960	}
2961
2962	if (wait_for_completion_interruptible(x: &ub->completion) != 0)
2963	return -EINTR;
2964
2965	if (!ublk_validate_user_pid(ub, ublksrv_pid))
2966	return -EINVAL;
2967
2968	mutex_lock(&ub->mutex);
2969	if (ub->dev_info.state == UBLK_S_DEV_LIVE ||
2970	test_bit(UB_STATE_USED, &ub->state)) {
2971	ret = -EEXIST;
2972	goto out_unlock;
2973	}
2974
2975	disk = blk_mq_alloc_disk(&ub->tag_set, &lim, NULL);
2976	if (IS_ERR(ptr: disk)) {
2977	ret = PTR_ERR(ptr: disk);
2978	goto out_unlock;
2979	}
2980	sprintf(buf: disk->disk_name, fmt: "ublkb%d", ub->ub_number);
2981	disk->fops = &ub_fops;
2982	disk->private_data = ub;
2983
2984	ub->dev_info.ublksrv_pid = ub->ublksrv_tgid;
2985	ub->ub_disk = disk;
2986
2987	ublk_apply_params(ub);
2988
2989	/*
2990	* Suppress partition scan to avoid potential IO hang.
2991	*
2992	* If ublk server error occurs during partition scan, the IO may
2993	* wait while holding ub->mutex, which can deadlock with other
2994	* operations that need the mutex. Defer partition scan to async
2995	* work.
2996	* For unprivileged daemons, keep GD_SUPPRESS_PART_SCAN set
2997	* permanently.
2998	*/
2999	set_bit(GD_SUPPRESS_PART_SCAN, addr: &disk->state);
3000
3001	ublk_get_device(ub);
3002	ub->dev_info.state = UBLK_S_DEV_LIVE;
3003
3004	if (ublk_dev_is_zoned(ub)) {
3005	ret = ublk_revalidate_disk_zones(ub);
3006	if (ret)
3007	goto out_put_cdev;
3008	}
3009
3010	ret = add_disk(disk);
3011	if (ret)
3012	goto out_put_cdev;
3013
3014	set_bit(UB_STATE_USED, addr: &ub->state);
3015
3016	/* Schedule async partition scan for trusted daemons */
3017	if (!ub->unprivileged_daemons)
3018	schedule_work(work: &ub->partition_scan_work);
3019
3020	out_put_cdev:
3021	if (ret) {
3022	ublk_detach_disk(ub);
3023	ublk_put_device(ub);
3024	}
3025	if (ret)
3026	put_disk(disk);
3027	out_unlock:
3028	mutex_unlock(lock: &ub->mutex);
3029	return ret;
3030	}
3031
3032	static int ublk_ctrl_get_queue_affinity(struct ublk_device *ub,
3033	const struct ublksrv_ctrl_cmd *header)
3034	{
3035	void __user *argp = (void __user *)(unsigned long)header->addr;
3036	cpumask_var_t cpumask;
3037	unsigned long queue;
3038	unsigned int retlen;
3039	unsigned int i;
3040	int ret;
3041
3042	if (header->len * BITS_PER_BYTE < nr_cpu_ids)
3043	return -EINVAL;
3044	if (header->len & (sizeof(unsigned long)-1))
3045	return -EINVAL;
3046	if (!header->addr)
3047	return -EINVAL;
3048
3049	queue = header->data[0];
3050	if (queue >= ub->dev_info.nr_hw_queues)
3051	return -EINVAL;
3052
3053	if (!zalloc_cpumask_var(mask: &cpumask, GFP_KERNEL))
3054	return -ENOMEM;
3055
3056	for_each_possible_cpu(i) {
3057	if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[i] == queue)
3058	cpumask_set_cpu(cpu: i, dstp: cpumask);
3059	}
3060
3061	ret = -EFAULT;
3062	retlen = min_t(unsigned short, header->len, cpumask_size());
3063	if (copy_to_user(to: argp, from: cpumask, n: retlen))
3064	goto out_free_cpumask;
3065	if (retlen != header->len &&
3066	clear_user(to: argp + retlen, n: header->len - retlen))
3067	goto out_free_cpumask;
3068
3069	ret = 0;
3070	out_free_cpumask:
3071	free_cpumask_var(mask: cpumask);
3072	return ret;
3073	}
3074
3075	static inline void ublk_dump_dev_info(struct ublksrv_ctrl_dev_info *info)
3076	{
3077	pr_devel("%s: dev id %d flags %llx\n", __func__,
3078	info->dev_id, info->flags);
3079	pr_devel("\t nr_hw_queues %d queue_depth %d\n",
3080	info->nr_hw_queues, info->queue_depth);
3081	}
3082
3083	static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header)
3084	{
3085	void __user *argp = (void __user *)(unsigned long)header->addr;
3086	struct ublksrv_ctrl_dev_info info;
3087	struct ublk_device *ub;
3088	int ret = -EINVAL;
3089
3090	if (header->len < sizeof(info) || !header->addr)
3091	return -EINVAL;
3092	if (header->queue_id != (u16)-1) {
3093	pr_warn("%s: queue_id is wrong %x\n",
3094	__func__, header->queue_id);
3095	return -EINVAL;
3096	}
3097
3098	if (copy_from_user(to: &info, from: argp, n: sizeof(info)))
3099	return -EFAULT;
3100
3101	if (info.queue_depth > UBLK_MAX_QUEUE_DEPTH || !info.queue_depth ||
3102	info.nr_hw_queues > UBLK_MAX_NR_QUEUES || !info.nr_hw_queues)
3103	return -EINVAL;
3104
3105	if (capable(CAP_SYS_ADMIN))
3106	info.flags &= ~UBLK_F_UNPRIVILEGED_DEV;
3107	else if (!(info.flags & UBLK_F_UNPRIVILEGED_DEV))
3108	return -EPERM;
3109
3110	/* forbid nonsense combinations of recovery flags */
3111	switch (info.flags & UBLK_F_ALL_RECOVERY_FLAGS) {
3112	case 0:
3113	case UBLK_F_USER_RECOVERY:
3114	case (UBLK_F_USER_RECOVERY | UBLK_F_USER_RECOVERY_REISSUE):
3115	case (UBLK_F_USER_RECOVERY | UBLK_F_USER_RECOVERY_FAIL_IO):
3116	break;
3117	default:
3118	pr_warn("%s: invalid recovery flags %llx\n", __func__,
3119	info.flags & UBLK_F_ALL_RECOVERY_FLAGS);
3120	return -EINVAL;
3121	}
3122
3123	if ((info.flags & UBLK_F_QUIESCE) && !(info.flags & UBLK_F_USER_RECOVERY)) {
3124	pr_warn("UBLK_F_QUIESCE requires UBLK_F_USER_RECOVERY\n");
3125	return -EINVAL;
3126	}
3127
3128	/*
3129	* unprivileged device can't be trusted, but RECOVERY and
3130	* RECOVERY_REISSUE still may hang error handling, so can't
3131	* support recovery features for unprivileged ublk now
3132	*
3133	* TODO: provide forward progress for RECOVERY handler, so that
3134	* unprivileged device can benefit from it
3135	*/
3136	if (info.flags & UBLK_F_UNPRIVILEGED_DEV) {
3137	info.flags &= ~(UBLK_F_USER_RECOVERY_REISSUE |
3138	UBLK_F_USER_RECOVERY);
3139
3140	/*
3141	* For USER_COPY, we depends on userspace to fill request
3142	* buffer by pwrite() to ublk char device, which can't be
3143	* used for unprivileged device
3144	*
3145	* Same with zero copy or auto buffer register.
3146	*/
3147	if (info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY |
3148	UBLK_F_AUTO_BUF_REG))
3149	return -EINVAL;
3150	}
3151
3152	/* the created device is always owned by current user */
3153	ublk_store_owner_uid_gid(owner_uid: &info.owner_uid, owner_gid: &info.owner_gid);
3154
3155	if (header->dev_id != info.dev_id) {
3156	pr_warn("%s: dev id not match %u %u\n",
3157	__func__, header->dev_id, info.dev_id);
3158	return -EINVAL;
3159	}
3160
3161	if (header->dev_id != U32_MAX && header->dev_id >= UBLK_MAX_UBLKS) {
3162	pr_warn("%s: dev id is too large. Max supported is %d\n",
3163	__func__, UBLK_MAX_UBLKS - 1);
3164	return -EINVAL;
3165	}
3166
3167	ublk_dump_dev_info(info: &info);
3168
3169	ret = mutex_lock_killable(&ublk_ctl_mutex);
3170	if (ret)
3171	return ret;
3172
3173	ret = -EACCES;
3174	if ((info.flags & UBLK_F_UNPRIVILEGED_DEV) &&
3175	unprivileged_ublks_added >= unprivileged_ublks_max)
3176	goto out_unlock;
3177
3178	ret = -ENOMEM;
3179	ub = kzalloc(struct_size(ub, queues, info.nr_hw_queues), GFP_KERNEL);
3180	if (!ub)
3181	goto out_unlock;
3182	mutex_init(&ub->mutex);
3183	spin_lock_init(&ub->lock);
3184	mutex_init(&ub->cancel_mutex);
3185	INIT_WORK(&ub->partition_scan_work, ublk_partition_scan_work);
3186
3187	ret = ublk_alloc_dev_number(ub, idx: header->dev_id);
3188	if (ret < 0)
3189	goto out_free_ub;
3190
3191	memcpy(&ub->dev_info, &info, sizeof(info));
3192
3193	/* update device id */
3194	ub->dev_info.dev_id = ub->ub_number;
3195
3196	/*
3197	* 64bit flags will be copied back to userspace as feature
3198	* negotiation result, so have to clear flags which driver
3199	* doesn't support yet, then userspace can get correct flags
3200	* (features) to handle.
3201	*/
3202	ub->dev_info.flags &= UBLK_F_ALL;
3203
3204	ub->dev_info.flags |= UBLK_F_CMD_IOCTL_ENCODE |
3205	UBLK_F_URING_CMD_COMP_IN_TASK |
3206	UBLK_F_PER_IO_DAEMON |
3207	UBLK_F_BUF_REG_OFF_DAEMON;
3208
3209	/* GET_DATA isn't needed any more with USER_COPY or ZERO COPY */
3210	if (ub->dev_info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY |
3211	UBLK_F_AUTO_BUF_REG))
3212	ub->dev_info.flags &= ~UBLK_F_NEED_GET_DATA;
3213
3214	/*
3215	* Zoned storage support requires reuse `ublksrv_io_cmd->addr` for
3216	* returning write_append_lba, which is only allowed in case of
3217	* user copy or zero copy
3218	*/
3219	if (ublk_dev_is_zoned(ub) &&
3220	(!IS_ENABLED(CONFIG_BLK_DEV_ZONED) || !(ub->dev_info.flags &
3221	(UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY)))) {
3222	ret = -EINVAL;
3223	goto out_free_dev_number;
3224	}
3225
3226	ub->dev_info.nr_hw_queues = min_t(unsigned int,
3227	ub->dev_info.nr_hw_queues, nr_cpu_ids);
3228	ublk_align_max_io_size(ub);
3229
3230	ret = ublk_add_tag_set(ub);
3231	if (ret)
3232	goto out_free_dev_number;
3233
3234	ret = ublk_init_queues(ub);
3235	if (ret)
3236	goto out_free_tag_set;
3237
3238	ret = -EFAULT;
3239	if (copy_to_user(to: argp, from: &ub->dev_info, n: sizeof(info)))
3240	goto out_deinit_queues;
3241
3242	/*
3243	* Add the char dev so that ublksrv daemon can be setup.
3244	* ublk_add_chdev() will cleanup everything if it fails.
3245	*/
3246	ret = ublk_add_chdev(ub);
3247	goto out_unlock;
3248
3249	out_deinit_queues:
3250	ublk_deinit_queues(ub);
3251	out_free_tag_set:
3252	blk_mq_free_tag_set(set: &ub->tag_set);
3253	out_free_dev_number:
3254	ublk_free_dev_number(ub);
3255	out_free_ub:
3256	mutex_destroy(lock: &ub->mutex);
3257	mutex_destroy(lock: &ub->cancel_mutex);
3258	kfree(objp: ub);
3259	out_unlock:
3260	mutex_unlock(lock: &ublk_ctl_mutex);
3261	return ret;
3262	}
3263
3264	static inline bool ublk_idr_freed(int id)
3265	{
3266	void *ptr;
3267
3268	spin_lock(lock: &ublk_idr_lock);
3269	ptr = idr_find(&ublk_index_idr, id);
3270	spin_unlock(lock: &ublk_idr_lock);
3271
3272	return ptr == NULL;
3273	}
3274
3275	static int ublk_ctrl_del_dev(struct ublk_device **p_ub, bool wait)
3276	{
3277	struct ublk_device *ub = *p_ub;
3278	int idx = ub->ub_number;
3279	int ret;
3280
3281	ret = mutex_lock_killable(&ublk_ctl_mutex);
3282	if (ret)
3283	return ret;
3284
3285	if (!test_bit(UB_STATE_DELETED, &ub->state)) {
3286	ublk_remove(ub);
3287	set_bit(UB_STATE_DELETED, addr: &ub->state);
3288	}
3289
3290	/* Mark the reference as consumed */
3291	*p_ub = NULL;
3292	ublk_put_device(ub);
3293	mutex_unlock(lock: &ublk_ctl_mutex);
3294
3295	/*
3296	* Wait until the idr is removed, then it can be reused after
3297	* DEL_DEV command is returned.
3298	*
3299	* If we returns because of user interrupt, future delete command
3300	* may come:
3301	*
3302	* - the device number isn't freed, this device won't or needn't
3303	* be deleted again, since UB_STATE_DELETED is set, and device
3304	* will be released after the last reference is dropped
3305	*
3306	* - the device number is freed already, we will not find this
3307	* device via ublk_get_device_from_id()
3308	*/
3309	if (wait && wait_event_interruptible(ublk_idr_wq, ublk_idr_freed(idx)))
3310	return -EINTR;
3311	return 0;
3312	}
3313
3314	static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd)
3315	{
3316	const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(sqe: cmd->sqe);
3317
3318	pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n",
3319	__func__, cmd->cmd_op, header->dev_id, header->queue_id,
3320	header->data[0], header->addr, header->len);
3321	}
3322
3323	static int ublk_ctrl_stop_dev(struct ublk_device *ub)
3324	{
3325	ublk_stop_dev(ub);
3326	return 0;
3327	}
3328
3329	static int ublk_ctrl_get_dev_info(struct ublk_device *ub,
3330	const struct ublksrv_ctrl_cmd *header)
3331	{
3332	struct task_struct *p;
3333	struct pid *pid;
3334	struct ublksrv_ctrl_dev_info dev_info;
3335	pid_t init_ublksrv_tgid = ub->dev_info.ublksrv_pid;
3336	void __user *argp = (void __user *)(unsigned long)header->addr;
3337
3338	if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr)
3339	return -EINVAL;
3340
3341	memcpy(&dev_info, &ub->dev_info, sizeof(dev_info));
3342	dev_info.ublksrv_pid = -1;
3343
3344	if (init_ublksrv_tgid > 0) {
3345	rcu_read_lock();
3346	pid = find_pid_ns(nr: init_ublksrv_tgid, ns: &init_pid_ns);
3347	p = pid_task(pid, PIDTYPE_TGID);
3348	if (p) {
3349	int vnr = task_tgid_vnr(tsk: p);
3350
3351	if (vnr)
3352	dev_info.ublksrv_pid = vnr;
3353	}
3354	rcu_read_unlock();
3355	}
3356
3357	if (copy_to_user(to: argp, from: &dev_info, n: sizeof(dev_info)))
3358	return -EFAULT;
3359
3360	return 0;
3361	}
3362
3363	/* TYPE_DEVT is readonly, so fill it up before returning to userspace */
3364	static void ublk_ctrl_fill_params_devt(struct ublk_device *ub)
3365	{
3366	ub->params.devt.char_major = MAJOR(ub->cdev_dev.devt);
3367	ub->params.devt.char_minor = MINOR(ub->cdev_dev.devt);
3368
3369	if (ub->ub_disk) {
3370	ub->params.devt.disk_major = MAJOR(disk_devt(ub->ub_disk));
3371	ub->params.devt.disk_minor = MINOR(disk_devt(ub->ub_disk));
3372	} else {
3373	ub->params.devt.disk_major = 0;
3374	ub->params.devt.disk_minor = 0;
3375	}
3376	ub->params.types |= UBLK_PARAM_TYPE_DEVT;
3377	}
3378
3379	static int ublk_ctrl_get_params(struct ublk_device *ub,
3380	const struct ublksrv_ctrl_cmd *header)
3381	{
3382	void __user *argp = (void __user *)(unsigned long)header->addr;
3383	struct ublk_params_header ph;
3384	int ret;
3385
3386	if (header->len <= sizeof(ph) || !header->addr)
3387	return -EINVAL;
3388
3389	if (copy_from_user(to: &ph, from: argp, n: sizeof(ph)))
3390	return -EFAULT;
3391
3392	if (ph.len > header->len || !ph.len)
3393	return -EINVAL;
3394
3395	if (ph.len > sizeof(struct ublk_params))
3396	ph.len = sizeof(struct ublk_params);
3397
3398	mutex_lock(&ub->mutex);
3399	ublk_ctrl_fill_params_devt(ub);
3400	if (copy_to_user(to: argp, from: &ub->params, n: ph.len))
3401	ret = -EFAULT;
3402	else
3403	ret = 0;
3404	mutex_unlock(lock: &ub->mutex);
3405
3406	return ret;
3407	}
3408
3409	static int ublk_ctrl_set_params(struct ublk_device *ub,
3410	const struct ublksrv_ctrl_cmd *header)
3411	{
3412	void __user *argp = (void __user *)(unsigned long)header->addr;
3413	struct ublk_params_header ph;
3414	int ret = -EFAULT;
3415
3416	if (header->len <= sizeof(ph) || !header->addr)
3417	return -EINVAL;
3418
3419	if (copy_from_user(to: &ph, from: argp, n: sizeof(ph)))
3420	return -EFAULT;
3421
3422	if (ph.len > header->len || !ph.len || !ph.types)
3423	return -EINVAL;
3424
3425	if (ph.len > sizeof(struct ublk_params))
3426	ph.len = sizeof(struct ublk_params);
3427
3428	mutex_lock(&ub->mutex);
3429	if (test_bit(UB_STATE_USED, &ub->state)) {
3430	/*
3431	* Parameters can only be changed when device hasn't
3432	* been started yet
3433	*/
3434	ret = -EACCES;
3435	} else if (copy_from_user(to: &ub->params, from: argp, n: ph.len)) {
3436	ret = -EFAULT;
3437	} else {
3438	/* clear all we don't support yet */
3439	ub->params.types &= UBLK_PARAM_TYPE_ALL;
3440	ret = ublk_validate_params(ub);
3441	if (ret)
3442	ub->params.types = 0;
3443	}
3444	mutex_unlock(lock: &ub->mutex);
3445
3446	return ret;
3447	}
3448
3449	static int ublk_ctrl_start_recovery(struct ublk_device *ub,
3450	const struct ublksrv_ctrl_cmd *header)
3451	{
3452	int ret = -EINVAL;
3453
3454	mutex_lock(&ub->mutex);
3455	if (ublk_nosrv_should_stop_dev(ub))
3456	goto out_unlock;
3457	/*
3458	* START_RECOVERY is only allowd after:
3459	*
3460	* (1) UB_STATE_OPEN is not set, which means the dying process is exited
3461	* and related io_uring ctx is freed so file struct of /dev/ublkcX is
3462	* released.
3463	*
3464	* and one of the following holds
3465	*
3466	* (2) UBLK_S_DEV_QUIESCED is set, which means the quiesce_work:
3467	* (a)has quiesced request queue
3468	* (b)has requeued every inflight rqs whose io_flags is ACTIVE
3469	* (c)has requeued/aborted every inflight rqs whose io_flags is NOT ACTIVE
3470	* (d)has completed/camceled all ioucmds owned by ther dying process
3471	*
3472	* (3) UBLK_S_DEV_FAIL_IO is set, which means the queue is not
3473	* quiesced, but all I/O is being immediately errored
3474	*/
3475	if (test_bit(UB_STATE_OPEN, &ub->state) || !ublk_dev_in_recoverable_state(ub)) {
3476	ret = -EBUSY;
3477	goto out_unlock;
3478	}
3479	pr_devel("%s: start recovery for dev id %d.\n", __func__, header->dev_id);
3480	init_completion(x: &ub->completion);
3481	ret = 0;
3482	out_unlock:
3483	mutex_unlock(lock: &ub->mutex);
3484	return ret;
3485	}
3486
3487	static int ublk_ctrl_end_recovery(struct ublk_device *ub,
3488	const struct ublksrv_ctrl_cmd *header)
3489	{
3490	int ublksrv_pid = (int)header->data[0];
3491	int ret = -EINVAL;
3492
3493	pr_devel("%s: Waiting for all FETCH_REQs, dev id %d...\n", __func__,
3494	header->dev_id);
3495
3496	if (wait_for_completion_interruptible(x: &ub->completion))
3497	return -EINTR;
3498
3499	pr_devel("%s: All FETCH_REQs received, dev id %d\n", __func__,
3500	header->dev_id);
3501
3502	if (!ublk_validate_user_pid(ub, ublksrv_pid))
3503	return -EINVAL;
3504
3505	mutex_lock(&ub->mutex);
3506	if (ublk_nosrv_should_stop_dev(ub))
3507	goto out_unlock;
3508
3509	if (!ublk_dev_in_recoverable_state(ub)) {
3510	ret = -EBUSY;
3511	goto out_unlock;
3512	}
3513	ub->dev_info.ublksrv_pid = ub->ublksrv_tgid;
3514	ub->dev_info.state = UBLK_S_DEV_LIVE;
3515	pr_devel("%s: new ublksrv_pid %d, dev id %d\n",
3516	__func__, ublksrv_pid, header->dev_id);
3517	blk_mq_kick_requeue_list(q: ub->ub_disk->queue);
3518	ret = 0;
3519	out_unlock:
3520	mutex_unlock(lock: &ub->mutex);
3521	return ret;
3522	}
3523
3524	static int ublk_ctrl_get_features(const struct ublksrv_ctrl_cmd *header)
3525	{
3526	void __user *argp = (void __user *)(unsigned long)header->addr;
3527	u64 features = UBLK_F_ALL;
3528
3529	if (header->len != UBLK_FEATURES_LEN || !header->addr)
3530	return -EINVAL;
3531
3532	if (copy_to_user(to: argp, from: &features, UBLK_FEATURES_LEN))
3533	return -EFAULT;
3534
3535	return 0;
3536	}
3537
3538	static void ublk_ctrl_set_size(struct ublk_device *ub, const struct ublksrv_ctrl_cmd *header)
3539	{
3540	struct ublk_param_basic *p = &ub->params.basic;
3541	u64 new_size = header->data[0];
3542
3543	mutex_lock(&ub->mutex);
3544	p->dev_sectors = new_size;
3545	set_capacity_and_notify(disk: ub->ub_disk, size: p->dev_sectors);
3546	mutex_unlock(lock: &ub->mutex);
3547	}
3548
3549	struct count_busy {
3550	const struct ublk_queue *ubq;
3551	unsigned int nr_busy;
3552	};
3553
3554	static bool ublk_count_busy_req(struct request *rq, void *data)
3555	{
3556	struct count_busy *idle = data;
3557
3558	if (!blk_mq_request_started(rq) && rq->mq_hctx->driver_data == idle->ubq)
3559	idle->nr_busy += 1;
3560	return true;
3561	}
3562
3563	/* uring_cmd is guaranteed to be active if the associated request is idle */
3564	static bool ubq_has_idle_io(const struct ublk_queue *ubq)
3565	{
3566	struct count_busy data = {
3567	.ubq = ubq,
3568	};
3569
3570	blk_mq_tagset_busy_iter(tagset: &ubq->dev->tag_set, fn: ublk_count_busy_req, priv: &data);
3571	return data.nr_busy < ubq->q_depth;
3572	}
3573
3574	/* Wait until each hw queue has at least one idle IO */
3575	static int ublk_wait_for_idle_io(struct ublk_device *ub,
3576	unsigned int timeout_ms)
3577	{
3578	unsigned int elapsed = 0;
3579	int ret;
3580
3581	while (elapsed < timeout_ms && !signal_pending(current)) {
3582	unsigned int queues_cancelable = 0;
3583	int i;
3584
3585	for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
3586	struct ublk_queue *ubq = ublk_get_queue(dev: ub, qid: i);
3587
3588	queues_cancelable += !!ubq_has_idle_io(ubq);
3589	}
3590
3591	/*
3592	* Each queue needs at least one active command for
3593	* notifying ublk server
3594	*/
3595	if (queues_cancelable == ub->dev_info.nr_hw_queues)
3596	break;
3597
3598	msleep(UBLK_REQUEUE_DELAY_MS);
3599	elapsed += UBLK_REQUEUE_DELAY_MS;
3600	}
3601
3602	if (signal_pending(current))
3603	ret = -EINTR;
3604	else if (elapsed >= timeout_ms)
3605	ret = -EBUSY;
3606	else
3607	ret = 0;
3608
3609	return ret;
3610	}
3611
3612	static int ublk_ctrl_quiesce_dev(struct ublk_device *ub,
3613	const struct ublksrv_ctrl_cmd *header)
3614	{
3615	/* zero means wait forever */
3616	u64 timeout_ms = header->data[0];
3617	struct gendisk *disk;
3618	int ret = -ENODEV;
3619
3620	if (!(ub->dev_info.flags & UBLK_F_QUIESCE))
3621	return -EOPNOTSUPP;
3622
3623	mutex_lock(&ub->mutex);
3624	disk = ublk_get_disk(ub);
3625	if (!disk)
3626	goto unlock;
3627	if (ub->dev_info.state == UBLK_S_DEV_DEAD)
3628	goto put_disk;
3629
3630	ret = 0;
3631	/* already in expected state */
3632	if (ub->dev_info.state != UBLK_S_DEV_LIVE)
3633	goto put_disk;
3634
3635	/* Mark the device as canceling */
3636	mutex_lock(&ub->cancel_mutex);
3637	blk_mq_quiesce_queue(q: disk->queue);
3638	ublk_set_canceling(ub, canceling: true);
3639	blk_mq_unquiesce_queue(q: disk->queue);
3640	mutex_unlock(lock: &ub->cancel_mutex);
3641
3642	if (!timeout_ms)
3643	timeout_ms = UINT_MAX;
3644	ret = ublk_wait_for_idle_io(ub, timeout_ms);
3645
3646	put_disk:
3647	ublk_put_disk(disk);
3648	unlock:
3649	mutex_unlock(lock: &ub->mutex);
3650
3651	/* Cancel pending uring_cmd */
3652	if (!ret)
3653	ublk_cancel_dev(ub);
3654	return ret;
3655	}
3656
3657	/*
3658	* All control commands are sent via /dev/ublk-control, so we have to check
3659	* the destination device's permission
3660	*/
3661	static int ublk_char_dev_permission(struct ublk_device *ub,
3662	const char *dev_path, int mask)
3663	{
3664	int err;
3665	struct path path;
3666	struct kstat stat;
3667
3668	err = kern_path(dev_path, LOOKUP_FOLLOW, &path);
3669	if (err)
3670	return err;
3671
3672	err = vfs_getattr(&path, &stat, STATX_TYPE, AT_STATX_SYNC_AS_STAT);
3673	if (err)
3674	goto exit;
3675
3676	err = -EPERM;
3677	if (stat.rdev != ub->cdev_dev.devt || !S_ISCHR(stat.mode))
3678	goto exit;
3679
3680	err = inode_permission(&nop_mnt_idmap,
3681	d_backing_inode(upper: path.dentry), mask);
3682	exit:
3683	path_put(&path);
3684	return err;
3685	}
3686
3687	static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,
3688	struct io_uring_cmd *cmd)
3689	{
3690	struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)io_uring_sqe_cmd(sqe: cmd->sqe);
3691	bool unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV;
3692	void __user *argp = (void __user *)(unsigned long)header->addr;
3693	char *dev_path = NULL;
3694	int ret = 0;
3695	int mask;
3696
3697	if (!unprivileged) {
3698	if (!capable(CAP_SYS_ADMIN))
3699	return -EPERM;
3700	/*
3701	* The new added command of UBLK_CMD_GET_DEV_INFO2 includes
3702	* char_dev_path in payload too, since userspace may not
3703	* know if the specified device is created as unprivileged
3704	* mode.
3705	*/
3706	if (_IOC_NR(cmd->cmd_op) != UBLK_CMD_GET_DEV_INFO2)
3707	return 0;
3708	}
3709
3710	/*
3711	* User has to provide the char device path for unprivileged ublk
3712	*
3713	* header->addr always points to the dev path buffer, and
3714	* header->dev_path_len records length of dev path buffer.
3715	*/
3716	if (!header->dev_path_len || header->dev_path_len > PATH_MAX)
3717	return -EINVAL;
3718
3719	if (header->len < header->dev_path_len)
3720	return -EINVAL;
3721
3722	dev_path = memdup_user_nul(argp, header->dev_path_len);
3723	if (IS_ERR(ptr: dev_path))
3724	return PTR_ERR(ptr: dev_path);
3725
3726	ret = -EINVAL;
3727	switch (_IOC_NR(cmd->cmd_op)) {
3728	case UBLK_CMD_GET_DEV_INFO:
3729	case UBLK_CMD_GET_DEV_INFO2:
3730	case UBLK_CMD_GET_QUEUE_AFFINITY:
3731	case UBLK_CMD_GET_PARAMS:
3732	case (_IOC_NR(UBLK_U_CMD_GET_FEATURES)):
3733	mask = MAY_READ;
3734	break;
3735	case UBLK_CMD_START_DEV:
3736	case UBLK_CMD_STOP_DEV:
3737	case UBLK_CMD_ADD_DEV:
3738	case UBLK_CMD_DEL_DEV:
3739	case UBLK_CMD_SET_PARAMS:
3740	case UBLK_CMD_START_USER_RECOVERY:
3741	case UBLK_CMD_END_USER_RECOVERY:
3742	case UBLK_CMD_UPDATE_SIZE:
3743	case UBLK_CMD_QUIESCE_DEV:
3744	mask = MAY_READ | MAY_WRITE;
3745	break;
3746	default:
3747	goto exit;
3748	}
3749
3750	ret = ublk_char_dev_permission(ub, dev_path, mask);
3751	if (!ret) {
3752	header->len -= header->dev_path_len;
3753	header->addr += header->dev_path_len;
3754	}
3755	pr_devel("%s: dev id %d cmd_op %x uid %d gid %d path %s ret %d\n",
3756	__func__, ub->ub_number, cmd->cmd_op,
3757	ub->dev_info.owner_uid, ub->dev_info.owner_gid,
3758	dev_path, ret);
3759	exit:
3760	kfree(objp: dev_path);
3761	return ret;
3762	}
3763
3764	static bool ublk_ctrl_uring_cmd_may_sleep(u32 cmd_op)
3765	{
3766	switch (_IOC_NR(cmd_op)) {
3767	case UBLK_CMD_GET_QUEUE_AFFINITY:
3768	case UBLK_CMD_GET_DEV_INFO:
3769	case UBLK_CMD_GET_DEV_INFO2:
3770	case _IOC_NR(UBLK_U_CMD_GET_FEATURES):
3771	return false;
3772	default:
3773	return true;
3774	}
3775	}
3776
3777	static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
3778	unsigned int issue_flags)
3779	{
3780	const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(sqe: cmd->sqe);
3781	struct ublk_device *ub = NULL;
3782	u32 cmd_op = cmd->cmd_op;
3783	int ret = -EINVAL;
3784
3785	if (ublk_ctrl_uring_cmd_may_sleep(cmd_op) &&
3786	issue_flags & IO_URING_F_NONBLOCK)
3787	return -EAGAIN;
3788
3789	ublk_ctrl_cmd_dump(cmd);
3790
3791	if (!(issue_flags & IO_URING_F_SQE128))
3792	goto out;
3793
3794	ret = ublk_check_cmd_op(cmd_op);
3795	if (ret)
3796	goto out;
3797
3798	if (cmd_op == UBLK_U_CMD_GET_FEATURES) {
3799	ret = ublk_ctrl_get_features(header);
3800	goto out;
3801	}
3802
3803	if (_IOC_NR(cmd_op) != UBLK_CMD_ADD_DEV) {
3804	ret = -ENODEV;
3805	ub = ublk_get_device_from_id(idx: header->dev_id);
3806	if (!ub)
3807	goto out;
3808
3809	ret = ublk_ctrl_uring_cmd_permission(ub, cmd);
3810	if (ret)
3811	goto put_dev;
3812	}
3813
3814	switch (_IOC_NR(cmd_op)) {
3815	case UBLK_CMD_START_DEV:
3816	ret = ublk_ctrl_start_dev(ub, header);
3817	break;
3818	case UBLK_CMD_STOP_DEV:
3819	ret = ublk_ctrl_stop_dev(ub);
3820	break;
3821	case UBLK_CMD_GET_DEV_INFO:
3822	case UBLK_CMD_GET_DEV_INFO2:
3823	ret = ublk_ctrl_get_dev_info(ub, header);
3824	break;
3825	case UBLK_CMD_ADD_DEV:
3826	ret = ublk_ctrl_add_dev(header);
3827	break;
3828	case UBLK_CMD_DEL_DEV:
3829	ret = ublk_ctrl_del_dev(p_ub: &ub, wait: true);
3830	break;
3831	case UBLK_CMD_DEL_DEV_ASYNC:
3832	ret = ublk_ctrl_del_dev(p_ub: &ub, wait: false);
3833	break;
3834	case UBLK_CMD_GET_QUEUE_AFFINITY:
3835	ret = ublk_ctrl_get_queue_affinity(ub, header);
3836	break;
3837	case UBLK_CMD_GET_PARAMS:
3838	ret = ublk_ctrl_get_params(ub, header);
3839	break;
3840	case UBLK_CMD_SET_PARAMS:
3841	ret = ublk_ctrl_set_params(ub, header);
3842	break;
3843	case UBLK_CMD_START_USER_RECOVERY:
3844	ret = ublk_ctrl_start_recovery(ub, header);
3845	break;
3846	case UBLK_CMD_END_USER_RECOVERY:
3847	ret = ublk_ctrl_end_recovery(ub, header);
3848	break;
3849	case UBLK_CMD_UPDATE_SIZE:
3850	ublk_ctrl_set_size(ub, header);
3851	ret = 0;
3852	break;
3853	case UBLK_CMD_QUIESCE_DEV:
3854	ret = ublk_ctrl_quiesce_dev(ub, header);
3855	break;
3856	default:
3857	ret = -EOPNOTSUPP;
3858	break;
3859	}
3860
3861	put_dev:
3862	if (ub)
3863	ublk_put_device(ub);
3864	out:
3865	pr_devel("%s: cmd done ret %d cmd_op %x, dev id %d qid %d\n",
3866	__func__, ret, cmd->cmd_op, header->dev_id, header->queue_id);
3867	return ret;
3868	}
3869
3870	static const struct file_operations ublk_ctl_fops = {
3871	.open = nonseekable_open,
3872	.uring_cmd = ublk_ctrl_uring_cmd,
3873	.owner = THIS_MODULE,
3874	.llseek = noop_llseek,
3875	};
3876
3877	static struct miscdevice ublk_misc = {
3878	.minor = MISC_DYNAMIC_MINOR,
3879	.name = "ublk-control",
3880	.fops = &ublk_ctl_fops,
3881	};
3882
3883	static int __init ublk_init(void)
3884	{
3885	int ret;
3886
3887	BUILD_BUG_ON((u64)UBLKSRV_IO_BUF_OFFSET +
3888	UBLKSRV_IO_BUF_TOTAL_SIZE < UBLKSRV_IO_BUF_OFFSET);
3889	BUILD_BUG_ON(sizeof(struct ublk_auto_buf_reg) != 8);
3890
3891	init_waitqueue_head(&ublk_idr_wq);
3892
3893	ret = misc_register(misc: &ublk_misc);
3894	if (ret)
3895	return ret;
3896
3897	ret = alloc_chrdev_region(&ublk_chr_devt, 0, UBLK_MINORS, "ublk-char");
3898	if (ret)
3899	goto unregister_mis;
3900
3901	ret = class_register(class: &ublk_chr_class);
3902	if (ret)
3903	goto free_chrdev_region;
3904
3905	return 0;
3906
3907	free_chrdev_region:
3908	unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS);
3909	unregister_mis:
3910	misc_deregister(misc: &ublk_misc);
3911	return ret;
3912	}
3913
3914	static void __exit ublk_exit(void)
3915	{
3916	struct ublk_device *ub;
3917	int id;
3918
3919	idr_for_each_entry(&ublk_index_idr, ub, id)
3920	ublk_remove(ub);
3921
3922	class_unregister(class: &ublk_chr_class);
3923	misc_deregister(misc: &ublk_misc);
3924
3925	idr_destroy(&ublk_index_idr);
3926	unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS);
3927	}
3928
3929	module_init(ublk_init);
3930	module_exit(ublk_exit);
3931
3932	static int ublk_set_max_unprivileged_ublks(const char *buf,
3933	const struct kernel_param *kp)
3934	{
3935	return param_set_uint_minmax(val: buf, kp, min: 0, UBLK_MAX_UBLKS);
3936	}
3937
3938	static int ublk_get_max_unprivileged_ublks(char *buf,
3939	const struct kernel_param *kp)
3940	{
3941	return sysfs_emit(buf, fmt: "%u\n", unprivileged_ublks_max);
3942	}
3943
3944	static const struct kernel_param_ops ublk_max_unprivileged_ublks_ops = {
3945	.set = ublk_set_max_unprivileged_ublks,
3946	.get = ublk_get_max_unprivileged_ublks,
3947	};
3948
3949	module_param_cb(ublks_max, &ublk_max_unprivileged_ublks_ops,
3950	&unprivileged_ublks_max, 0644);
3951	MODULE_PARM_DESC(ublks_max, "max number of unprivileged ublk devices allowed to add(default: 64)");
3952
3953	MODULE_AUTHOR("Ming Lei <ming.lei@redhat.com>");
3954	MODULE_DESCRIPTION("Userspace block device");
3955	MODULE_LICENSE("GPL");
3956