-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathscsi_lib.c
More file actions
3578 lines (3158 loc) · 93.6 KB
/
scsi_lib.c
File metadata and controls
3578 lines (3158 loc) · 93.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 1999 Eric Youngdale
* Copyright (C) 2014 Christoph Hellwig
*
* SCSI queueing library.
* Initial versions: Eric Youngdale ([email protected]).
* Based upon conversations with large numbers
* of people at Linux Expo.
*/
#include <linux/bio.h>
#include <linux/bitops.h>
#include <linux/blkdev.h>
#include <linux/completion.h>
#include <linux/ctype.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/hardirq.h>
#include <linux/scatterlist.h>
#include <linux/blk-mq.h>
#include <linux/blk-integrity.h>
#include <linux/ratelimit.h>
#include <linux/unaligned.h>
#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_driver.h>
#include <scsi/scsi_eh.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport.h> /* scsi_init_limits() */
#include <scsi/scsi_dh.h>
#include <trace/events/scsi.h>
#include "scsi_debugfs.h"
#include "scsi_priv.h"
#include "scsi_logging.h"
/*
* Size of integrity metadata is usually small, 1 inline sg should
* cover normal cases.
*/
#ifdef CONFIG_ARCH_NO_SG_CHAIN
#define SCSI_INLINE_PROT_SG_CNT 0
#define SCSI_INLINE_SG_CNT 0
#else
#define SCSI_INLINE_PROT_SG_CNT 1
#define SCSI_INLINE_SG_CNT 2
#endif
static struct kmem_cache *scsi_sense_cache;
static DEFINE_MUTEX(scsi_sense_cache_mutex);
static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd);
int scsi_init_sense_cache(struct Scsi_Host *shost)
{
int ret = 0;
mutex_lock(&scsi_sense_cache_mutex);
if (!scsi_sense_cache) {
scsi_sense_cache =
kmem_cache_create_usercopy("scsi_sense_cache",
SCSI_SENSE_BUFFERSIZE, 0, SLAB_HWCACHE_ALIGN,
0, SCSI_SENSE_BUFFERSIZE, NULL);
if (!scsi_sense_cache)
ret = -ENOMEM;
}
mutex_unlock(&scsi_sense_cache_mutex);
return ret;
}
static void
scsi_set_blocked(struct scsi_cmnd *cmd, enum scsi_qc_status reason)
{
struct Scsi_Host *host = cmd->device->host;
struct scsi_device *device = cmd->device;
struct scsi_target *starget = scsi_target(device);
/*
* Set the appropriate busy bit for the device/host.
*
* If the host/device isn't busy, assume that something actually
* completed, and that we should be able to queue a command now.
*
* Note that the prior mid-layer assumption that any host could
* always queue at least one command is now broken. The mid-layer
* will implement a user specifiable stall (see
* scsi_host.max_host_blocked and scsi_device.max_device_blocked)
* if a command is requeued with no other commands outstanding
* either for the device or for the host.
*/
switch (reason) {
case SCSI_MLQUEUE_HOST_BUSY:
atomic_set(&host->host_blocked, host->max_host_blocked);
break;
case SCSI_MLQUEUE_DEVICE_BUSY:
case SCSI_MLQUEUE_EH_RETRY:
atomic_set(&device->device_blocked,
device->max_device_blocked);
break;
case SCSI_MLQUEUE_TARGET_BUSY:
atomic_set(&starget->target_blocked,
starget->max_target_blocked);
break;
}
}
static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd, unsigned long msecs)
{
struct request *rq = scsi_cmd_to_rq(cmd);
if (rq->rq_flags & RQF_DONTPREP) {
rq->rq_flags &= ~RQF_DONTPREP;
scsi_mq_uninit_cmd(cmd);
} else {
WARN_ON_ONCE(true);
}
blk_mq_requeue_request(rq, false);
if (!scsi_host_in_recovery(cmd->device->host))
blk_mq_delay_kick_requeue_list(rq->q, msecs);
}
/**
* __scsi_queue_insert - private queue insertion
* @cmd: The SCSI command being requeued
* @reason: The reason for the requeue
* @unbusy: Whether the queue should be unbusied
*
* This is a private queue insertion. The public interface
* scsi_queue_insert() always assumes the queue should be unbusied
* because it's always called before the completion. This function is
* for a requeue after completion, which should only occur in this
* file.
*/
static void __scsi_queue_insert(struct scsi_cmnd *cmd,
enum scsi_qc_status reason, bool unbusy)
{
struct scsi_device *device = cmd->device;
SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd,
"Inserting command %p into mlqueue\n", cmd));
scsi_set_blocked(cmd, reason);
/*
* Decrement the counters, since these commands are no longer
* active on the host/device.
*/
if (unbusy)
scsi_device_unbusy(device, cmd);
/*
* Requeue this command. It will go before all other commands
* that are already in the queue. Schedule requeue work under
* lock such that the kblockd_schedule_work() call happens
* before blk_mq_destroy_queue() finishes.
*/
cmd->result = 0;
blk_mq_requeue_request(scsi_cmd_to_rq(cmd),
!scsi_host_in_recovery(cmd->device->host));
}
/**
* scsi_queue_insert - Reinsert a command in the queue.
* @cmd: command that we are adding to queue.
* @reason: why we are inserting command to queue.
*
* We do this for one of two cases. Either the host is busy and it cannot accept
* any more commands for the time being, or the device returned QUEUE_FULL and
* can accept no more commands.
*
* Context: This could be called either from an interrupt context or a normal
* process context.
*/
void scsi_queue_insert(struct scsi_cmnd *cmd, enum scsi_qc_status reason)
{
__scsi_queue_insert(cmd, reason, true);
}
/**
* scsi_failures_reset_retries - reset all failures to zero
* @failures: &struct scsi_failures with specific failure modes set
*/
void scsi_failures_reset_retries(struct scsi_failures *failures)
{
struct scsi_failure *failure;
failures->total_retries = 0;
for (failure = failures->failure_definitions; failure->result;
failure++)
failure->retries = 0;
}
EXPORT_SYMBOL_GPL(scsi_failures_reset_retries);
/**
* scsi_check_passthrough - Determine if passthrough scsi_cmnd needs a retry.
* @scmd: scsi_cmnd to check.
* @failures: scsi_failures struct that lists failures to check for.
*
* Returns -EAGAIN if the caller should retry else 0.
*/
static int scsi_check_passthrough(struct scsi_cmnd *scmd,
struct scsi_failures *failures)
{
struct scsi_failure *failure;
struct scsi_sense_hdr sshdr;
enum sam_status status;
if (!scmd->result)
return 0;
if (!failures)
return 0;
for (failure = failures->failure_definitions; failure->result;
failure++) {
if (failure->result == SCMD_FAILURE_RESULT_ANY)
goto maybe_retry;
if (host_byte(scmd->result) &&
host_byte(scmd->result) == host_byte(failure->result))
goto maybe_retry;
status = status_byte(scmd->result);
if (!status)
continue;
if (failure->result == SCMD_FAILURE_STAT_ANY &&
!scsi_status_is_good(scmd->result))
goto maybe_retry;
if (status != status_byte(failure->result))
continue;
if (status_byte(failure->result) != SAM_STAT_CHECK_CONDITION ||
failure->sense == SCMD_FAILURE_SENSE_ANY)
goto maybe_retry;
if (!scsi_command_normalize_sense(scmd, &sshdr))
return 0;
if (failure->sense != sshdr.sense_key)
continue;
if (failure->asc == SCMD_FAILURE_ASC_ANY)
goto maybe_retry;
if (failure->asc != sshdr.asc)
continue;
if (failure->ascq == SCMD_FAILURE_ASCQ_ANY ||
failure->ascq == sshdr.ascq)
goto maybe_retry;
}
return 0;
maybe_retry:
if (failure->allowed) {
if (failure->allowed == SCMD_FAILURE_NO_LIMIT ||
++failure->retries <= failure->allowed)
return -EAGAIN;
} else {
if (failures->total_allowed == SCMD_FAILURE_NO_LIMIT ||
++failures->total_retries <= failures->total_allowed)
return -EAGAIN;
}
return 0;
}
/**
* scsi_execute_cmd - insert request and wait for the result
* @sdev: scsi_device
* @cmd: scsi command
* @opf: block layer request cmd_flags
* @buffer: data buffer
* @bufflen: len of buffer
* @timeout: request timeout in HZ
* @ml_retries: number of times SCSI midlayer will retry request
* @args: Optional args. See struct definition for field descriptions
*
* Returns the scsi_cmnd result field if a command was executed, or a negative
* Linux error code if we didn't get that far.
*/
int scsi_execute_cmd(struct scsi_device *sdev, const unsigned char *cmd,
blk_opf_t opf, void *buffer, unsigned int bufflen,
int timeout, int ml_retries,
const struct scsi_exec_args *args)
{
static const struct scsi_exec_args default_args;
struct request *req;
struct scsi_cmnd *scmd;
int ret;
if (!args)
args = &default_args;
else if (WARN_ON_ONCE(args->sense &&
args->sense_len != SCSI_SENSE_BUFFERSIZE))
return -EINVAL;
retry:
req = scsi_alloc_request(sdev->request_queue, opf, args->req_flags);
if (IS_ERR(req))
return PTR_ERR(req);
if (bufflen) {
ret = blk_rq_map_kern(req, buffer, bufflen, GFP_NOIO);
if (ret)
goto out;
}
scmd = blk_mq_rq_to_pdu(req);
scmd->cmd_len = COMMAND_SIZE(cmd[0]);
memcpy(scmd->cmnd, cmd, scmd->cmd_len);
scmd->allowed = ml_retries;
scmd->flags |= args->scmd_flags;
req->timeout = timeout;
req->rq_flags |= RQF_QUIET;
/*
* head injection *required* here otherwise quiesce won't work
*/
blk_execute_rq(req, true);
if (scsi_check_passthrough(scmd, args->failures) == -EAGAIN) {
blk_mq_free_request(req);
goto retry;
}
/*
* Some devices (USB mass-storage in particular) may transfer
* garbage data together with a residue indicating that the data
* is invalid. Prevent the garbage from being misinterpreted
* and prevent security leaks by zeroing out the excess data.
*/
if (unlikely(scmd->resid_len > 0 && scmd->resid_len <= bufflen))
memset(buffer + bufflen - scmd->resid_len, 0, scmd->resid_len);
if (args->resid)
*args->resid = scmd->resid_len;
if (args->sense)
memcpy(args->sense, scmd->sense_buffer, SCSI_SENSE_BUFFERSIZE);
if (args->sshdr)
scsi_normalize_sense(scmd->sense_buffer, scmd->sense_len,
args->sshdr);
ret = scmd->result;
out:
blk_mq_free_request(req);
return ret;
}
EXPORT_SYMBOL(scsi_execute_cmd);
/*
* Wake up the error handler if necessary. Avoid as follows that the error
* handler is not woken up if host in-flight requests number ==
* shost->host_failed: use call_rcu() in scsi_eh_scmd_add() in combination
* with an RCU read lock in this function to ensure that this function in
* its entirety either finishes before scsi_eh_scmd_add() increases the
* host_failed counter or that it notices the shost state change made by
* scsi_eh_scmd_add().
*/
static void scsi_dec_host_busy(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
{
unsigned long flags;
rcu_read_lock();
__clear_bit(SCMD_STATE_INFLIGHT, &cmd->state);
if (unlikely(scsi_host_in_recovery(shost))) {
/*
* Ensure the clear of SCMD_STATE_INFLIGHT is visible to
* other CPUs before counting busy requests. Otherwise,
* reordering can cause CPUs to race and miss an eh wakeup
* when no CPU sees all busy requests as done or timed out.
*/
smp_mb();
unsigned int busy = scsi_host_busy(shost);
spin_lock_irqsave(shost->host_lock, flags);
if (shost->host_failed || shost->host_eh_scheduled)
scsi_eh_wakeup(shost, busy);
spin_unlock_irqrestore(shost->host_lock, flags);
}
rcu_read_unlock();
}
void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd)
{
struct Scsi_Host *shost = sdev->host;
struct scsi_target *starget = scsi_target(sdev);
scsi_dec_host_busy(shost, cmd);
if (starget->can_queue > 0)
atomic_dec(&starget->target_busy);
if (sdev->budget_map.map)
sbitmap_put(&sdev->budget_map, cmd->budget_token);
cmd->budget_token = -1;
}
/*
* Kick the queue of SCSI device @sdev if @sdev != current_sdev. Called with
* interrupts disabled.
*/
static void scsi_kick_sdev_queue(struct scsi_device *sdev, void *data)
{
struct scsi_device *current_sdev = data;
if (sdev != current_sdev)
blk_mq_run_hw_queues(sdev->request_queue, true);
}
/*
* Called for single_lun devices on IO completion. Clear starget_sdev_user,
* and call blk_run_queue for all the scsi_devices on the target -
* including current_sdev first.
*
* Called with *no* scsi locks held.
*/
static void scsi_single_lun_run(struct scsi_device *current_sdev)
{
struct Scsi_Host *shost = current_sdev->host;
struct scsi_target *starget = scsi_target(current_sdev);
unsigned long flags;
spin_lock_irqsave(shost->host_lock, flags);
starget->starget_sdev_user = NULL;
spin_unlock_irqrestore(shost->host_lock, flags);
/*
* Call blk_run_queue for all LUNs on the target, starting with
* current_sdev. We race with others (to set starget_sdev_user),
* but in most cases, we will be first. Ideally, each LU on the
* target would get some limited time or requests on the target.
*/
blk_mq_run_hw_queues(current_sdev->request_queue,
shost->queuecommand_may_block);
spin_lock_irqsave(shost->host_lock, flags);
if (!starget->starget_sdev_user)
__starget_for_each_device(starget, current_sdev,
scsi_kick_sdev_queue);
spin_unlock_irqrestore(shost->host_lock, flags);
}
static inline bool scsi_device_is_busy(struct scsi_device *sdev)
{
if (scsi_device_busy(sdev) >= sdev->queue_depth)
return true;
if (atomic_read(&sdev->device_blocked) > 0)
return true;
return false;
}
static inline bool scsi_target_is_busy(struct scsi_target *starget)
{
if (starget->can_queue > 0) {
if (atomic_read(&starget->target_busy) >= starget->can_queue)
return true;
if (atomic_read(&starget->target_blocked) > 0)
return true;
}
return false;
}
static inline bool scsi_host_is_busy(struct Scsi_Host *shost)
{
if (atomic_read(&shost->host_blocked) > 0)
return true;
if (shost->host_self_blocked)
return true;
return false;
}
static void scsi_starved_list_run(struct Scsi_Host *shost)
{
LIST_HEAD(starved_list);
struct scsi_device *sdev;
unsigned long flags;
spin_lock_irqsave(shost->host_lock, flags);
list_splice_init(&shost->starved_list, &starved_list);
while (!list_empty(&starved_list)) {
struct request_queue *slq;
/*
* As long as shost is accepting commands and we have
* starved queues, call blk_run_queue. scsi_request_fn
* drops the queue_lock and can add us back to the
* starved_list.
*
* host_lock protects the starved_list and starved_entry.
* scsi_request_fn must get the host_lock before checking
* or modifying starved_list or starved_entry.
*/
if (scsi_host_is_busy(shost))
break;
sdev = list_entry(starved_list.next,
struct scsi_device, starved_entry);
list_del_init(&sdev->starved_entry);
if (scsi_target_is_busy(scsi_target(sdev))) {
list_move_tail(&sdev->starved_entry,
&shost->starved_list);
continue;
}
/*
* Once we drop the host lock, a racing scsi_remove_device()
* call may remove the sdev from the starved list and destroy
* it and the queue. Mitigate by taking a reference to the
* queue and never touching the sdev again after we drop the
* host lock. Note: if __scsi_remove_device() invokes
* blk_mq_destroy_queue() before the queue is run from this
* function then blk_run_queue() will return immediately since
* blk_mq_destroy_queue() marks the queue with QUEUE_FLAG_DYING.
*/
slq = sdev->request_queue;
if (!blk_get_queue(slq))
continue;
spin_unlock_irqrestore(shost->host_lock, flags);
blk_mq_run_hw_queues(slq, false);
blk_put_queue(slq);
spin_lock_irqsave(shost->host_lock, flags);
}
/* put any unprocessed entries back */
list_splice(&starved_list, &shost->starved_list);
spin_unlock_irqrestore(shost->host_lock, flags);
}
/**
* scsi_run_queue - Select a proper request queue to serve next.
* @q: last request's queue
*
* The previous command was completely finished, start a new one if possible.
*/
static void scsi_run_queue(struct request_queue *q)
{
struct scsi_device *sdev = q->queuedata;
if (scsi_target(sdev)->single_lun)
scsi_single_lun_run(sdev);
if (!list_empty(&sdev->host->starved_list))
scsi_starved_list_run(sdev->host);
/* Note: blk_mq_kick_requeue_list() runs the queue asynchronously. */
blk_mq_kick_requeue_list(q);
}
void scsi_requeue_run_queue(struct work_struct *work)
{
struct scsi_device *sdev;
struct request_queue *q;
sdev = container_of(work, struct scsi_device, requeue_work);
q = sdev->request_queue;
scsi_run_queue(q);
}
void scsi_run_host_queues(struct Scsi_Host *shost)
{
struct scsi_device *sdev;
shost_for_each_device(sdev, shost)
scsi_run_queue(sdev->request_queue);
}
static void scsi_uninit_cmd(struct scsi_cmnd *cmd)
{
if (!blk_rq_is_passthrough(scsi_cmd_to_rq(cmd))) {
struct scsi_driver *drv = scsi_cmd_to_driver(cmd);
if (drv->uninit_command)
drv->uninit_command(cmd);
}
}
void scsi_free_sgtables(struct scsi_cmnd *cmd)
{
if (cmd->sdb.table.nents)
sg_free_table_chained(&cmd->sdb.table,
SCSI_INLINE_SG_CNT);
if (scsi_prot_sg_count(cmd))
sg_free_table_chained(&cmd->prot_sdb->table,
SCSI_INLINE_PROT_SG_CNT);
}
EXPORT_SYMBOL_GPL(scsi_free_sgtables);
static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)
{
scsi_free_sgtables(cmd);
scsi_uninit_cmd(cmd);
}
static void scsi_run_queue_async(struct scsi_device *sdev)
{
if (scsi_host_in_recovery(sdev->host))
return;
if (scsi_target(sdev)->single_lun ||
!list_empty(&sdev->host->starved_list)) {
kblockd_schedule_work(&sdev->requeue_work);
} else {
/*
* smp_mb() present in sbitmap_queue_clear() or implied in
* .end_io is for ordering writing .device_busy in
* scsi_device_unbusy() and reading sdev->restarts.
*/
int old = atomic_read(&sdev->restarts);
/*
* ->restarts has to be kept as non-zero if new budget
* contention occurs.
*
* No need to run queue when either another re-run
* queue wins in updating ->restarts or a new budget
* contention occurs.
*/
if (old && atomic_cmpxchg(&sdev->restarts, old, 0) == old)
blk_mq_run_hw_queues(sdev->request_queue, true);
}
}
/* Returns false when no more bytes to process, true if there are more */
static bool scsi_end_request(struct request *req, blk_status_t error,
unsigned int bytes)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
struct scsi_device *sdev = cmd->device;
struct request_queue *q = sdev->request_queue;
if (blk_update_request(req, error, bytes))
return true;
if (q->limits.features & BLK_FEAT_ADD_RANDOM)
add_disk_randomness(req->q->disk);
WARN_ON_ONCE(!blk_rq_is_passthrough(req) &&
!(cmd->flags & SCMD_INITIALIZED));
cmd->flags = 0;
/*
* Calling rcu_barrier() is not necessary here because the
* SCSI error handler guarantees that the function called by
* call_rcu() has been called before scsi_end_request() is
* called.
*/
destroy_rcu_head(&cmd->rcu);
/*
* In the MQ case the command gets freed by __blk_mq_end_request,
* so we have to do all cleanup that depends on it earlier.
*
* We also can't kick the queues from irq context, so we
* will have to defer it to a workqueue.
*/
scsi_mq_uninit_cmd(cmd);
/*
* queue is still alive, so grab the ref for preventing it
* from being cleaned up during running queue.
*/
percpu_ref_get(&q->q_usage_counter);
__blk_mq_end_request(req, error);
scsi_run_queue_async(sdev);
percpu_ref_put(&q->q_usage_counter);
return false;
}
/**
* scsi_result_to_blk_status - translate a SCSI result code into blk_status_t
* @result: scsi error code
*
* Translate a SCSI result code into a blk_status_t value.
*/
static blk_status_t scsi_result_to_blk_status(int result)
{
/*
* Check the scsi-ml byte first in case we converted a host or status
* byte.
*/
switch (scsi_ml_byte(result)) {
case SCSIML_STAT_OK:
break;
case SCSIML_STAT_RESV_CONFLICT:
return BLK_STS_RESV_CONFLICT;
case SCSIML_STAT_NOSPC:
return BLK_STS_NOSPC;
case SCSIML_STAT_MED_ERROR:
return BLK_STS_MEDIUM;
case SCSIML_STAT_TGT_FAILURE:
return BLK_STS_TARGET;
case SCSIML_STAT_DL_TIMEOUT:
return BLK_STS_DURATION_LIMIT;
}
switch (host_byte(result)) {
case DID_OK:
if (scsi_status_is_good(result))
return BLK_STS_OK;
return BLK_STS_IOERR;
case DID_TRANSPORT_FAILFAST:
case DID_TRANSPORT_MARGINAL:
return BLK_STS_TRANSPORT;
default:
return BLK_STS_IOERR;
}
}
/**
* scsi_rq_err_bytes - determine number of bytes till the next failure boundary
* @rq: request to examine
*
* Description:
* A request could be merge of IOs which require different failure
* handling. This function determines the number of bytes which
* can be failed from the beginning of the request without
* crossing into area which need to be retried further.
*
* Return:
* The number of bytes to fail.
*/
static unsigned int scsi_rq_err_bytes(const struct request *rq)
{
blk_opf_t ff = rq->cmd_flags & REQ_FAILFAST_MASK;
unsigned int bytes = 0;
struct bio *bio;
if (!(rq->rq_flags & RQF_MIXED_MERGE))
return blk_rq_bytes(rq);
/*
* Currently the only 'mixing' which can happen is between
* different fastfail types. We can safely fail portions
* which have all the failfast bits that the first one has -
* the ones which are at least as eager to fail as the first
* one.
*/
for (bio = rq->bio; bio; bio = bio->bi_next) {
if ((bio->bi_opf & ff) != ff)
break;
bytes += bio->bi_iter.bi_size;
}
/* this could lead to infinite loop */
BUG_ON(blk_rq_bytes(rq) && !bytes);
return bytes;
}
static bool scsi_cmd_runtime_exceeced(struct scsi_cmnd *cmd)
{
struct request *req = scsi_cmd_to_rq(cmd);
unsigned long wait_for;
if (cmd->allowed == SCSI_CMD_RETRIES_NO_LIMIT)
return false;
wait_for = (cmd->allowed + 1) * req->timeout;
if (time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
scmd_printk(KERN_ERR, cmd, "timing out command, waited %lus\n",
wait_for/HZ);
return true;
}
return false;
}
/*
* When ALUA transition state is returned, reprep the cmd to
* use the ALUA handler's transition timeout. Delay the reprep
* 1 sec to avoid aggressive retries of the target in that
* state.
*/
#define ALUA_TRANSITION_REPREP_DELAY 1000
/* Helper for scsi_io_completion() when special action required. */
static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result)
{
struct request *req = scsi_cmd_to_rq(cmd);
int level = 0;
enum {ACTION_FAIL, ACTION_REPREP, ACTION_DELAYED_REPREP,
ACTION_RETRY, ACTION_DELAYED_RETRY} action;
struct scsi_sense_hdr sshdr;
bool sense_valid;
bool sense_current = true; /* false implies "deferred sense" */
blk_status_t blk_stat;
sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
if (sense_valid)
sense_current = !scsi_sense_is_deferred(&sshdr);
blk_stat = scsi_result_to_blk_status(result);
if (host_byte(result) == DID_RESET) {
/* Third party bus reset or reset for error recovery
* reasons. Just retry the command and see what
* happens.
*/
action = ACTION_RETRY;
} else if (sense_valid && sense_current) {
switch (sshdr.sense_key) {
case UNIT_ATTENTION:
if (cmd->device->removable) {
/* Detected disc change. Set a bit
* and quietly refuse further access.
*/
cmd->device->changed = 1;
action = ACTION_FAIL;
} else {
/* Must have been a power glitch, or a
* bus reset. Could not have been a
* media change, so we just retry the
* command and see what happens.
*/
action = ACTION_RETRY;
}
break;
case ILLEGAL_REQUEST:
/* If we had an ILLEGAL REQUEST returned, then
* we may have performed an unsupported
* command. The only thing this should be
* would be a ten byte read where only a six
* byte read was supported. Also, on a system
* where READ CAPACITY failed, we may have
* read past the end of the disk.
*/
if ((cmd->device->use_10_for_rw &&
sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
(cmd->cmnd[0] == READ_10 ||
cmd->cmnd[0] == WRITE_10)) {
/* This will issue a new 6-byte command. */
cmd->device->use_10_for_rw = 0;
action = ACTION_REPREP;
} else if (sshdr.asc == 0x10) /* DIX */ {
action = ACTION_FAIL;
blk_stat = BLK_STS_PROTECTION;
/* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */
} else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) {
action = ACTION_FAIL;
blk_stat = BLK_STS_TARGET;
} else
action = ACTION_FAIL;
break;
case ABORTED_COMMAND:
action = ACTION_FAIL;
if (sshdr.asc == 0x10) /* DIF */
blk_stat = BLK_STS_PROTECTION;
break;
case NOT_READY:
/* If the device is in the process of becoming
* ready, or has a temporary blockage, retry.
*/
if (sshdr.asc == 0x04) {
switch (sshdr.ascq) {
case 0x01: /* becoming ready */
case 0x04: /* format in progress */
case 0x05: /* rebuild in progress */
case 0x06: /* recalculation in progress */
case 0x07: /* operation in progress */
case 0x08: /* Long write in progress */
case 0x09: /* self test in progress */
case 0x11: /* notify (enable spinup) required */
case 0x14: /* space allocation in progress */
case 0x1a: /* start stop unit in progress */
case 0x1b: /* sanitize in progress */
case 0x1d: /* configuration in progress */
action = ACTION_DELAYED_RETRY;
break;
case 0x0a: /* ALUA state transition */
action = ACTION_DELAYED_REPREP;
break;
/*
* Depopulation might take many hours,
* thus it is not worthwhile to retry.
*/
case 0x24: /* depopulation in progress */
case 0x25: /* depopulation restore in progress */
fallthrough;
default:
action = ACTION_FAIL;
break;
}
} else
action = ACTION_FAIL;
break;
case VOLUME_OVERFLOW:
/* See SSC3rXX or current. */
action = ACTION_FAIL;
break;
case DATA_PROTECT:
action = ACTION_FAIL;
if ((sshdr.asc == 0x0C && sshdr.ascq == 0x12) ||
(sshdr.asc == 0x55 &&
(sshdr.ascq == 0x0E || sshdr.ascq == 0x0F))) {
/* Insufficient zone resources */
blk_stat = BLK_STS_ZONE_OPEN_RESOURCE;
}
break;
case COMPLETED:
fallthrough;
default:
action = ACTION_FAIL;
break;
}
} else
action = ACTION_FAIL;
if (action != ACTION_FAIL && scsi_cmd_runtime_exceeced(cmd))
action = ACTION_FAIL;
switch (action) {
case ACTION_FAIL:
/* Give up and fail the remainder of the request */
if (!(req->rq_flags & RQF_QUIET)) {
static DEFINE_RATELIMIT_STATE(_rs,
DEFAULT_RATELIMIT_INTERVAL,
DEFAULT_RATELIMIT_BURST);
if (unlikely(scsi_logging_level))
level =
SCSI_LOG_LEVEL(SCSI_LOG_MLCOMPLETE_SHIFT,
SCSI_LOG_MLCOMPLETE_BITS);
/*
* if logging is enabled the failure will be printed
* in scsi_log_completion(), so avoid duplicate messages
*/
if (!level && __ratelimit(&_rs)) {
scsi_print_result(cmd, NULL, FAILED);
if (sense_valid)
scsi_print_sense(cmd);
scsi_print_command(cmd);
}
}
if (!scsi_end_request(req, blk_stat, scsi_rq_err_bytes(req)))
return;
fallthrough;
case ACTION_REPREP:
scsi_mq_requeue_cmd(cmd, 0);
break;
case ACTION_DELAYED_REPREP:
scsi_mq_requeue_cmd(cmd, ALUA_TRANSITION_REPREP_DELAY);
break;
case ACTION_RETRY:
/* Retry the same command immediately */
__scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, false);
break;
case ACTION_DELAYED_RETRY:
/* Retry the same command after a delay */
__scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, false);
break;
}
}
/*
* Helper for scsi_io_completion() when cmd->result is non-zero. Returns a
* new result that may suppress further error checking. Also modifies
* *blk_statp in some cases.
*/
static int scsi_io_completion_nz_result(struct scsi_cmnd *cmd, int result,
blk_status_t *blk_statp)
{
bool sense_valid;
bool sense_current = true; /* false implies "deferred sense" */
struct request *req = scsi_cmd_to_rq(cmd);
struct scsi_sense_hdr sshdr;
sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
if (sense_valid)
sense_current = !scsi_sense_is_deferred(&sshdr);
if (blk_rq_is_passthrough(req)) {
if (sense_valid) {
/*
* SG_IO wants current and deferred errors
*/
cmd->sense_len = min(8 + cmd->sense_buffer[7],
SCSI_SENSE_BUFFERSIZE);
}
if (sense_current)
*blk_statp = scsi_result_to_blk_status(result);
} else if (blk_rq_bytes(req) == 0 && sense_current) {