@@ -1140,6 +1140,25 @@ static void ublk_complete_rq(struct kref *ref)
11401140 __ublk_complete_rq (req );
11411141}
11421142
1143+ static void ublk_do_fail_rq (struct request * req )
1144+ {
1145+ struct ublk_queue * ubq = req -> mq_hctx -> driver_data ;
1146+
1147+ if (ublk_nosrv_should_reissue_outstanding (ubq -> dev ))
1148+ blk_mq_requeue_request (req , false);
1149+ else
1150+ __ublk_complete_rq (req );
1151+ }
1152+
1153+ static void ublk_fail_rq_fn (struct kref * ref )
1154+ {
1155+ struct ublk_rq_data * data = container_of (ref , struct ublk_rq_data ,
1156+ ref );
1157+ struct request * req = blk_mq_rq_from_pdu (data );
1158+
1159+ ublk_do_fail_rq (req );
1160+ }
1161+
11431162/*
11441163 * Since ublk_rq_task_work_cb always fails requests immediately during
11451164 * exiting, __ublk_fail_req() is only called from abort context during
@@ -1153,10 +1172,13 @@ static void __ublk_fail_req(struct ublk_queue *ubq, struct ublk_io *io,
11531172{
11541173 WARN_ON_ONCE (io -> flags & UBLK_IO_FLAG_ACTIVE );
11551174
1156- if (ublk_nosrv_should_reissue_outstanding (ubq -> dev ))
1157- blk_mq_requeue_request (req , false);
1158- else
1159- ublk_put_req_ref (ubq , req );
1175+ if (ublk_need_req_ref (ubq )) {
1176+ struct ublk_rq_data * data = blk_mq_rq_to_pdu (req );
1177+
1178+ kref_put (& data -> ref , ublk_fail_rq_fn );
1179+ } else {
1180+ ublk_do_fail_rq (req );
1181+ }
11601182}
11611183
11621184static void ubq_complete_io_cmd (struct ublk_io * io , int res ,
@@ -1349,7 +1371,8 @@ static enum blk_eh_timer_return ublk_timeout(struct request *rq)
13491371 return BLK_EH_RESET_TIMER ;
13501372}
13511373
1352- static blk_status_t ublk_prep_req (struct ublk_queue * ubq , struct request * rq )
1374+ static blk_status_t ublk_prep_req (struct ublk_queue * ubq , struct request * rq ,
1375+ bool check_cancel )
13531376{
13541377 blk_status_t res ;
13551378
@@ -1368,7 +1391,7 @@ static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq)
13681391 if (ublk_nosrv_should_queue_io (ubq ) && unlikely (ubq -> force_abort ))
13691392 return BLK_STS_IOERR ;
13701393
1371- if (unlikely (ubq -> canceling ))
1394+ if (check_cancel && unlikely (ubq -> canceling ))
13721395 return BLK_STS_IOERR ;
13731396
13741397 /* fill iod to slot in io cmd buffer */
@@ -1387,7 +1410,7 @@ static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
13871410 struct request * rq = bd -> rq ;
13881411 blk_status_t res ;
13891412
1390- res = ublk_prep_req (ubq , rq );
1413+ res = ublk_prep_req (ubq , rq , false );
13911414 if (res != BLK_STS_OK )
13921415 return res ;
13931416
@@ -1419,7 +1442,7 @@ static void ublk_queue_rqs(struct rq_list *rqlist)
14191442 ublk_queue_cmd_list (ubq , & submit_list );
14201443 ubq = this_q ;
14211444
1422- if (ublk_prep_req (ubq , req ) == BLK_STS_OK )
1445+ if (ublk_prep_req (ubq , req , true ) == BLK_STS_OK )
14231446 rq_list_add_tail (& submit_list , req );
14241447 else
14251448 rq_list_add_tail (& requeue_list , req );
@@ -2413,9 +2436,9 @@ static struct ublk_device *ublk_get_device_from_id(int idx)
24132436 return ub ;
24142437}
24152438
2416- static int ublk_ctrl_start_dev (struct ublk_device * ub , struct io_uring_cmd * cmd )
2439+ static int ublk_ctrl_start_dev (struct ublk_device * ub ,
2440+ const struct ublksrv_ctrl_cmd * header )
24172441{
2418- const struct ublksrv_ctrl_cmd * header = io_uring_sqe_cmd (cmd -> sqe );
24192442 const struct ublk_param_basic * p = & ub -> params .basic ;
24202443 int ublksrv_pid = (int )header -> data [0 ];
24212444 struct queue_limits lim = {
@@ -2534,9 +2557,8 @@ static int ublk_ctrl_start_dev(struct ublk_device *ub, struct io_uring_cmd *cmd)
25342557}
25352558
25362559static int ublk_ctrl_get_queue_affinity (struct ublk_device * ub ,
2537- struct io_uring_cmd * cmd )
2560+ const struct ublksrv_ctrl_cmd * header )
25382561{
2539- const struct ublksrv_ctrl_cmd * header = io_uring_sqe_cmd (cmd -> sqe );
25402562 void __user * argp = (void __user * )(unsigned long )header -> addr ;
25412563 cpumask_var_t cpumask ;
25422564 unsigned long queue ;
@@ -2585,9 +2607,8 @@ static inline void ublk_dump_dev_info(struct ublksrv_ctrl_dev_info *info)
25852607 info -> nr_hw_queues , info -> queue_depth );
25862608}
25872609
2588- static int ublk_ctrl_add_dev (struct io_uring_cmd * cmd )
2610+ static int ublk_ctrl_add_dev (const struct ublksrv_ctrl_cmd * header )
25892611{
2590- const struct ublksrv_ctrl_cmd * header = io_uring_sqe_cmd (cmd -> sqe );
25912612 void __user * argp = (void __user * )(unsigned long )header -> addr ;
25922613 struct ublksrv_ctrl_dev_info info ;
25932614 struct ublk_device * ub ;
@@ -2812,9 +2833,8 @@ static int ublk_ctrl_stop_dev(struct ublk_device *ub)
28122833}
28132834
28142835static int ublk_ctrl_get_dev_info (struct ublk_device * ub ,
2815- struct io_uring_cmd * cmd )
2836+ const struct ublksrv_ctrl_cmd * header )
28162837{
2817- const struct ublksrv_ctrl_cmd * header = io_uring_sqe_cmd (cmd -> sqe );
28182838 void __user * argp = (void __user * )(unsigned long )header -> addr ;
28192839
28202840 if (header -> len < sizeof (struct ublksrv_ctrl_dev_info ) || !header -> addr )
@@ -2843,9 +2863,8 @@ static void ublk_ctrl_fill_params_devt(struct ublk_device *ub)
28432863}
28442864
28452865static int ublk_ctrl_get_params (struct ublk_device * ub ,
2846- struct io_uring_cmd * cmd )
2866+ const struct ublksrv_ctrl_cmd * header )
28472867{
2848- const struct ublksrv_ctrl_cmd * header = io_uring_sqe_cmd (cmd -> sqe );
28492868 void __user * argp = (void __user * )(unsigned long )header -> addr ;
28502869 struct ublk_params_header ph ;
28512870 int ret ;
@@ -2874,9 +2893,8 @@ static int ublk_ctrl_get_params(struct ublk_device *ub,
28742893}
28752894
28762895static int ublk_ctrl_set_params (struct ublk_device * ub ,
2877- struct io_uring_cmd * cmd )
2896+ const struct ublksrv_ctrl_cmd * header )
28782897{
2879- const struct ublksrv_ctrl_cmd * header = io_uring_sqe_cmd (cmd -> sqe );
28802898 void __user * argp = (void __user * )(unsigned long )header -> addr ;
28812899 struct ublk_params_header ph ;
28822900 int ret = - EFAULT ;
@@ -2940,9 +2958,8 @@ static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq)
29402958}
29412959
29422960static int ublk_ctrl_start_recovery (struct ublk_device * ub ,
2943- struct io_uring_cmd * cmd )
2961+ const struct ublksrv_ctrl_cmd * header )
29442962{
2945- const struct ublksrv_ctrl_cmd * header = io_uring_sqe_cmd (cmd -> sqe );
29462963 int ret = - EINVAL ;
29472964 int i ;
29482965
@@ -2988,9 +3005,8 @@ static int ublk_ctrl_start_recovery(struct ublk_device *ub,
29883005}
29893006
29903007static int ublk_ctrl_end_recovery (struct ublk_device * ub ,
2991- struct io_uring_cmd * cmd )
3008+ const struct ublksrv_ctrl_cmd * header )
29923009{
2993- const struct ublksrv_ctrl_cmd * header = io_uring_sqe_cmd (cmd -> sqe );
29943010 int ublksrv_pid = (int )header -> data [0 ];
29953011 int ret = - EINVAL ;
29963012 int i ;
@@ -3037,9 +3053,8 @@ static int ublk_ctrl_end_recovery(struct ublk_device *ub,
30373053 return ret ;
30383054}
30393055
3040- static int ublk_ctrl_get_features (struct io_uring_cmd * cmd )
3056+ static int ublk_ctrl_get_features (const struct ublksrv_ctrl_cmd * header )
30413057{
3042- const struct ublksrv_ctrl_cmd * header = io_uring_sqe_cmd (cmd -> sqe );
30433058 void __user * argp = (void __user * )(unsigned long )header -> addr ;
30443059 u64 features = UBLK_F_ALL ;
30453060
@@ -3178,7 +3193,7 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
31783193 goto out ;
31793194
31803195 if (cmd_op == UBLK_U_CMD_GET_FEATURES ) {
3181- ret = ublk_ctrl_get_features (cmd );
3196+ ret = ublk_ctrl_get_features (header );
31823197 goto out ;
31833198 }
31843199
@@ -3195,17 +3210,17 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
31953210
31963211 switch (_IOC_NR (cmd_op )) {
31973212 case UBLK_CMD_START_DEV :
3198- ret = ublk_ctrl_start_dev (ub , cmd );
3213+ ret = ublk_ctrl_start_dev (ub , header );
31993214 break ;
32003215 case UBLK_CMD_STOP_DEV :
32013216 ret = ublk_ctrl_stop_dev (ub );
32023217 break ;
32033218 case UBLK_CMD_GET_DEV_INFO :
32043219 case UBLK_CMD_GET_DEV_INFO2 :
3205- ret = ublk_ctrl_get_dev_info (ub , cmd );
3220+ ret = ublk_ctrl_get_dev_info (ub , header );
32063221 break ;
32073222 case UBLK_CMD_ADD_DEV :
3208- ret = ublk_ctrl_add_dev (cmd );
3223+ ret = ublk_ctrl_add_dev (header );
32093224 break ;
32103225 case UBLK_CMD_DEL_DEV :
32113226 ret = ublk_ctrl_del_dev (& ub , true);
@@ -3214,19 +3229,19 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
32143229 ret = ublk_ctrl_del_dev (& ub , false);
32153230 break ;
32163231 case UBLK_CMD_GET_QUEUE_AFFINITY :
3217- ret = ublk_ctrl_get_queue_affinity (ub , cmd );
3232+ ret = ublk_ctrl_get_queue_affinity (ub , header );
32183233 break ;
32193234 case UBLK_CMD_GET_PARAMS :
3220- ret = ublk_ctrl_get_params (ub , cmd );
3235+ ret = ublk_ctrl_get_params (ub , header );
32213236 break ;
32223237 case UBLK_CMD_SET_PARAMS :
3223- ret = ublk_ctrl_set_params (ub , cmd );
3238+ ret = ublk_ctrl_set_params (ub , header );
32243239 break ;
32253240 case UBLK_CMD_START_USER_RECOVERY :
3226- ret = ublk_ctrl_start_recovery (ub , cmd );
3241+ ret = ublk_ctrl_start_recovery (ub , header );
32273242 break ;
32283243 case UBLK_CMD_END_USER_RECOVERY :
3229- ret = ublk_ctrl_end_recovery (ub , cmd );
3244+ ret = ublk_ctrl_end_recovery (ub , header );
32303245 break ;
32313246 default :
32323247 ret = - EOPNOTSUPP ;
0 commit comments