Allow some commands to run while taking lock

We need to execute commands like RTPG and Inquiry while taking the lock
because if taking it takes a while we could run out of retries and the
initiator will fail to add the path back automatically.

Note:

For handlers implementing both handle_cmd and read/write callbacks, the
handle_cmd callback now works like the passthrough only handlers.
It must now call tcmu_dev_in_recovery before accessing the data in
tcmur_dev_get_private, or it must do its own locking/refcounting with
the data stored there, or only call tcmur lib functions which will do
the right thing. Upstream handlers have been converted to do the latter.

Signed-off-by: Mike Christie <mchristi@redhat.com>

tcmur_cmd_handler.c

@@ -785,6 +785,9 @@ int tcmur_handle_writesame(struct tcmu_device *dev, struct tcmulib_cmd *cmd,
 	struct tcmur_handler *rhandler = tcmu_get_runner_handler(dev);
 	int ret;
 
+	if (tcmu_dev_in_recovery(dev))
+		return TCMU_STS_BUSY;
+
 	ret = alua_check_state(dev, cmd);
 	if (ret)
 		return ret;
@@ -1807,6 +1810,9 @@ int tcmur_handle_caw(struct tcmu_device *dev, struct tcmulib_cmd *cmd,
 	int ret;
 	uint8_t sectors = cmd->cdb[13];
 
+	if (tcmu_dev_in_recovery(dev))
+		return TCMU_STS_BUSY;
+
 	/* From sbc4r12a section 5.3 COMPARE AND WRITE command
 	 * A NUMBER OF LOGICAL BLOCKS field set to zero specifies that no
 	 * read operations shall be performed, no logical block data shall
@@ -2406,12 +2412,7 @@ static int handle_try_passthrough(struct tcmu_device *dev,
 
 	track_aio_request_start(rdev);
 
-	if (tcmu_dev_in_recovery(dev)) {
-		ret = TCMU_STS_BUSY;
-	} else {
-		ret = rhandler->handle_cmd(dev, cmd);
-	}
-
+	ret = rhandler->handle_cmd(dev, cmd);
 	if (ret != TCMU_STS_ASYNC_HANDLED)
 		track_aio_request_finish(rdev, NULL);
 

tcmur_device.c

@@ -45,13 +45,7 @@ int __tcmu_reopen_dev(struct tcmu_device *dev, int retries)
 	bool needs_close = false;
 	bool cancel_lock = false;
 
-	tcmu_dev_dbg(dev, "Waiting for outstanding commands to complete\n");
-	ret = aio_wait_for_empty_queue(rdev);
-
 	pthread_mutex_lock(&rdev->state_lock);
-	if (ret)
-		goto done;
-
 	if (rdev->flags & TCMUR_DEV_FLAG_STOPPING) {
 		ret = 0;
 		goto done;
@@ -84,6 +78,21 @@ int __tcmu_reopen_dev(struct tcmu_device *dev, int retries)
 	rdev->flags &= ~TCMUR_DEV_FLAG_IS_OPEN;
 	pthread_mutex_unlock(&rdev->state_lock);
 
+	if (pthread_self() != rdev->cmdproc_thread)
+		/*
+		 * The cmdproc thread could be starting to execute a new IO.
+		 * Make sure sync cmd handler callbacks for cmds like INQUIRY
+		 * are completed.
+		 */
+		tcmu_flush_device(dev);
+
+	tcmu_dev_dbg(dev, "Waiting for outstanding commands to complete\n");
+	ret = aio_wait_for_empty_queue(rdev);
+	if (ret) {
+		pthread_mutex_lock(&rdev->state_lock);
+		goto done;
+	}
+
 	if (needs_close) {
 		tcmu_dev_dbg(dev, "Closing device.\n");
 		rhandler->close(dev);
@@ -361,24 +370,12 @@ int tcmu_acquire_dev_lock(struct tcmu_device *dev, bool is_sync,
 	int retries = 0, ret = TCMU_STS_HW_ERR;
 	bool reopen;
 
-	/* Block the kernel device. */
-	tcmu_block_device(dev);
-
 	tcmu_dev_dbg(dev, "Waiting for outstanding commands to complete\n");
 	if (aio_wait_for_empty_queue(rdev)) {
 		tcmu_dev_err(dev, "Not able to flush queue before taking lock.\n");
 		goto done;
 	}
 
-	/*
-	 * Handle race where cmd could be in tcmur_generic_handle_cmd before
-	 * the aio handler. For explicit ALUA, we execute the lock call from
-	 * the main io processing thread, so this will deadlock waiting on
-	 * the STPG.
-	 */
-	if (!is_sync)
-		tcmu_flush_device(dev);
-
 	reopen = false;
 	pthread_mutex_lock(&rdev->state_lock);
 	if (rdev->lock_lost || !(rdev->flags & TCMUR_DEV_FLAG_IS_OPEN)) {
@@ -423,18 +420,28 @@ int tcmu_acquire_dev_lock(struct tcmu_device *dev, bool is_sync,
 	}
 
 done:
+	/* Block and flush stale IO from the kernel device and ring. */
+	tcmu_block_device(dev);
+	/*
+	 * Handle race where cmd could be in tcmur_generic_handle_cmd before
+	 * the aio handler. For explicit ALUA, we execute the lock call from
+	 * the main io processing thread, so we only flush here for implicit.
+	 */
+	if (!is_sync)
+		tcmu_flush_device(dev);
+
 	/* TODO: set UA based on bgly's patches */
 	pthread_mutex_lock(&rdev->state_lock);
 	if (ret == TCMU_STS_OK)
 		rdev->lock_state = TCMUR_DEV_LOCK_LOCKED;
 	else
 		rdev->lock_state = TCMUR_DEV_LOCK_UNLOCKED;
 	tcmu_dev_dbg(dev, "lock call done. lock state %d\n", rdev->lock_state);
+	tcmu_unblock_device(dev);
+
 	pthread_cond_signal(&rdev->lock_cond);
 	pthread_mutex_unlock(&rdev->state_lock);
 
-	tcmu_unblock_device(dev);
-
 	return ret;
 }