LKML Archive on lore.kernel.org
help / color / mirror / Atom feed
* [PATCH v2] mmc: dw_mmc: Add data CRC error injection
@ 2021-07-01  8:05 ` Vincent Whitchurch
  2021-07-25 21:41   ` Jaehoon Chung
  2021-08-04 11:34   ` Ulf Hansson
  0 siblings, 2 replies; 3+ messages in thread
From: Vincent Whitchurch @ 2021-07-01  8:05 UTC (permalink / raw)
  To: Jaehoon Chung, Ulf Hansson
  Cc: kernel, Vincent Whitchurch, linux-mmc, linux-kernel

This driver has had problems when handling data errors.  Add fault
injection support so that the abort handling can be easily triggered and
regression-tested.  A hrtimer is used to indicate a data CRC error at
various points during the data transfer.

Note that for the recent problem with hangs in the case of some data CRC
errors, a udelay(10) inserted at the start of send_stop_abort() greatly
helped in triggering the error, but I've not included this as part of
the fault injection support since it seemed too specific.

Signed-off-by: Vincent Whitchurch <vincent.whitchurch@axis.com>
---

Notes:
    v2: Add missing includes.

 drivers/mmc/host/dw_mmc.c | 73 +++++++++++++++++++++++++++++++++++++++
 drivers/mmc/host/dw_mmc.h |  7 ++++
 2 files changed, 80 insertions(+)

diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index d333130d1531..dbbb94e6ff4b 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -17,9 +17,11 @@
 #include <linux/interrupt.h>
 #include <linux/iopoll.h>
 #include <linux/ioport.h>
+#include <linux/ktime.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/prandom.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/stat.h>
@@ -181,6 +183,9 @@ static void dw_mci_init_debugfs(struct dw_mci_slot *slot)
 			   &host->pending_events);
 	debugfs_create_xul("completed_events", S_IRUSR, root,
 			   &host->completed_events);
+#ifdef CONFIG_FAULT_INJECTION
+	fault_create_debugfs_attr("fail_data_crc", root, &host->fail_data_crc);
+#endif
 }
 #endif /* defined(CONFIG_DEBUG_FS) */
 
@@ -1788,6 +1793,68 @@ static const struct mmc_host_ops dw_mci_ops = {
 	.prepare_hs400_tuning	= dw_mci_prepare_hs400_tuning,
 };
 
+#ifdef CONFIG_FAULT_INJECTION
+static enum hrtimer_restart dw_mci_fault_timer(struct hrtimer *t)
+{
+	struct dw_mci *host = container_of(t, struct dw_mci, fault_timer);
+	unsigned long flags;
+
+	spin_lock_irqsave(&host->irq_lock, flags);
+
+	if (!host->data_status)
+		host->data_status = SDMMC_INT_DCRC;
+	set_bit(EVENT_DATA_ERROR, &host->pending_events);
+	tasklet_schedule(&host->tasklet);
+
+	spin_unlock_irqrestore(&host->irq_lock, flags);
+
+	return HRTIMER_NORESTART;
+}
+
+static void dw_mci_start_fault_timer(struct dw_mci *host)
+{
+	struct mmc_data *data = host->data;
+
+	if (!data || data->blocks <= 1)
+		return;
+
+	if (!should_fail(&host->fail_data_crc, 1))
+		return;
+
+	/*
+	 * Try to inject the error at random points during the data transfer.
+	 */
+	hrtimer_start(&host->fault_timer,
+		      ms_to_ktime(prandom_u32() % 25),
+		      HRTIMER_MODE_REL);
+}
+
+static void dw_mci_stop_fault_timer(struct dw_mci *host)
+{
+	hrtimer_cancel(&host->fault_timer);
+}
+
+static void dw_mci_init_fault(struct dw_mci *host)
+{
+	host->fail_data_crc = (struct fault_attr) FAULT_ATTR_INITIALIZER;
+
+	hrtimer_init(&host->fault_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	host->fault_timer.function = dw_mci_fault_timer;
+}
+#else
+static void dw_mci_init_fault(struct dw_mci *host)
+{
+}
+
+static void dw_mci_start_fault_timer(struct dw_mci *host)
+{
+}
+
+static void dw_mci_stop_fault_timer(struct dw_mci *host)
+{
+}
+#endif
+
 static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq)
 	__releases(&host->lock)
 	__acquires(&host->lock)
@@ -2102,6 +2169,7 @@ static void dw_mci_tasklet_func(struct tasklet_struct *t)
 				break;
 			}
 
+			dw_mci_stop_fault_timer(host);
 			host->data = NULL;
 			set_bit(EVENT_DATA_COMPLETE, &host->completed_events);
 			err = dw_mci_data_complete(host, data);
@@ -2151,6 +2219,7 @@ static void dw_mci_tasklet_func(struct tasklet_struct *t)
 			if (mrq->cmd->error && mrq->data)
 				dw_mci_reset(host);
 
+			dw_mci_stop_fault_timer(host);
 			host->cmd = NULL;
 			host->data = NULL;
 
@@ -2600,6 +2669,8 @@ static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status)
 
 	set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
 	tasklet_schedule(&host->tasklet);
+
+	dw_mci_start_fault_timer(host);
 }
 
 static void dw_mci_handle_cd(struct dw_mci *host)
@@ -3223,6 +3294,8 @@ int dw_mci_probe(struct dw_mci *host)
 	spin_lock_init(&host->irq_lock);
 	INIT_LIST_HEAD(&host->queue);
 
+	dw_mci_init_fault(host);
+
 	/*
 	 * Get the host data width - this assumes that HCON has been set with
 	 * the correct values.
diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
index da5923a92e60..ce05d81477d9 100644
--- a/drivers/mmc/host/dw_mmc.h
+++ b/drivers/mmc/host/dw_mmc.h
@@ -14,6 +14,8 @@
 #include <linux/mmc/core.h>
 #include <linux/dmaengine.h>
 #include <linux/reset.h>
+#include <linux/fault-inject.h>
+#include <linux/hrtimer.h>
 #include <linux/interrupt.h>
 
 enum dw_mci_state {
@@ -230,6 +232,11 @@ struct dw_mci {
 	struct timer_list       cmd11_timer;
 	struct timer_list       cto_timer;
 	struct timer_list       dto_timer;
+
+#ifdef CONFIG_FAULT_INJECTION
+	struct fault_attr	fail_data_crc;
+	struct hrtimer		fault_timer;
+#endif
 };
 
 /* DMA ops for Internal/External DMAC interface */
-- 
2.28.0


^ permalink raw reply	[flat|nested] 3+ messages in thread

* Re: [PATCH v2] mmc: dw_mmc: Add data CRC error injection
  2021-07-01  8:05 ` [PATCH v2] mmc: dw_mmc: Add data CRC error injection Vincent Whitchurch
@ 2021-07-25 21:41   ` Jaehoon Chung
  2021-08-04 11:34   ` Ulf Hansson
  1 sibling, 0 replies; 3+ messages in thread
From: Jaehoon Chung @ 2021-07-25 21:41 UTC (permalink / raw)
  To: Vincent Whitchurch, Ulf Hansson; +Cc: kernel, linux-mmc, linux-kernel

On 7/1/21 5:05 PM, Vincent Whitchurch wrote:
> This driver has had problems when handling data errors.  Add fault
> injection support so that the abort handling can be easily triggered and
> regression-tested.  A hrtimer is used to indicate a data CRC error at
> various points during the data transfer.
> 
> Note that for the recent problem with hangs in the case of some data CRC
> errors, a udelay(10) inserted at the start of send_stop_abort() greatly
> helped in triggering the error, but I've not included this as part of
> the fault injection support since it seemed too specific.
> 
> Signed-off-by: Vincent Whitchurch <vincent.whitchurch@axis.com>

Reviewed-by: Jaehoon Chung <jh80.chung@samsung.com>

Best Regards,
Jaehoon Chung

> ---
> 
> Notes:
>     v2: Add missing includes.
> 
>  drivers/mmc/host/dw_mmc.c | 73 +++++++++++++++++++++++++++++++++++++++
>  drivers/mmc/host/dw_mmc.h |  7 ++++
>  2 files changed, 80 insertions(+)
> 
> diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
> index d333130d1531..dbbb94e6ff4b 100644
> --- a/drivers/mmc/host/dw_mmc.c
> +++ b/drivers/mmc/host/dw_mmc.c
> @@ -17,9 +17,11 @@
>  #include <linux/interrupt.h>
>  #include <linux/iopoll.h>
>  #include <linux/ioport.h>
> +#include <linux/ktime.h>
>  #include <linux/module.h>
>  #include <linux/platform_device.h>
>  #include <linux/pm_runtime.h>
> +#include <linux/prandom.h>
>  #include <linux/seq_file.h>
>  #include <linux/slab.h>
>  #include <linux/stat.h>
> @@ -181,6 +183,9 @@ static void dw_mci_init_debugfs(struct dw_mci_slot *slot)
>  			   &host->pending_events);
>  	debugfs_create_xul("completed_events", S_IRUSR, root,
>  			   &host->completed_events);
> +#ifdef CONFIG_FAULT_INJECTION
> +	fault_create_debugfs_attr("fail_data_crc", root, &host->fail_data_crc);
> +#endif
>  }
>  #endif /* defined(CONFIG_DEBUG_FS) */
>  
> @@ -1788,6 +1793,68 @@ static const struct mmc_host_ops dw_mci_ops = {
>  	.prepare_hs400_tuning	= dw_mci_prepare_hs400_tuning,
>  };
>  
> +#ifdef CONFIG_FAULT_INJECTION
> +static enum hrtimer_restart dw_mci_fault_timer(struct hrtimer *t)
> +{
> +	struct dw_mci *host = container_of(t, struct dw_mci, fault_timer);
> +	unsigned long flags;
> +
> +	spin_lock_irqsave(&host->irq_lock, flags);
> +
> +	if (!host->data_status)
> +		host->data_status = SDMMC_INT_DCRC;
> +	set_bit(EVENT_DATA_ERROR, &host->pending_events);
> +	tasklet_schedule(&host->tasklet);
> +
> +	spin_unlock_irqrestore(&host->irq_lock, flags);
> +
> +	return HRTIMER_NORESTART;
> +}
> +
> +static void dw_mci_start_fault_timer(struct dw_mci *host)
> +{
> +	struct mmc_data *data = host->data;
> +
> +	if (!data || data->blocks <= 1)
> +		return;
> +
> +	if (!should_fail(&host->fail_data_crc, 1))
> +		return;
> +
> +	/*
> +	 * Try to inject the error at random points during the data transfer.
> +	 */
> +	hrtimer_start(&host->fault_timer,
> +		      ms_to_ktime(prandom_u32() % 25),
> +		      HRTIMER_MODE_REL);
> +}
> +
> +static void dw_mci_stop_fault_timer(struct dw_mci *host)
> +{
> +	hrtimer_cancel(&host->fault_timer);
> +}
> +
> +static void dw_mci_init_fault(struct dw_mci *host)
> +{
> +	host->fail_data_crc = (struct fault_attr) FAULT_ATTR_INITIALIZER;
> +
> +	hrtimer_init(&host->fault_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
> +	host->fault_timer.function = dw_mci_fault_timer;
> +}
> +#else
> +static void dw_mci_init_fault(struct dw_mci *host)
> +{
> +}
> +
> +static void dw_mci_start_fault_timer(struct dw_mci *host)
> +{
> +}
> +
> +static void dw_mci_stop_fault_timer(struct dw_mci *host)
> +{
> +}
> +#endif
> +
>  static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq)
>  	__releases(&host->lock)
>  	__acquires(&host->lock)
> @@ -2102,6 +2169,7 @@ static void dw_mci_tasklet_func(struct tasklet_struct *t)
>  				break;
>  			}
>  
> +			dw_mci_stop_fault_timer(host);
>  			host->data = NULL;
>  			set_bit(EVENT_DATA_COMPLETE, &host->completed_events);
>  			err = dw_mci_data_complete(host, data);
> @@ -2151,6 +2219,7 @@ static void dw_mci_tasklet_func(struct tasklet_struct *t)
>  			if (mrq->cmd->error && mrq->data)
>  				dw_mci_reset(host);
>  
> +			dw_mci_stop_fault_timer(host);
>  			host->cmd = NULL;
>  			host->data = NULL;
>  
> @@ -2600,6 +2669,8 @@ static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status)
>  
>  	set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
>  	tasklet_schedule(&host->tasklet);
> +
> +	dw_mci_start_fault_timer(host);
>  }
>  
>  static void dw_mci_handle_cd(struct dw_mci *host)
> @@ -3223,6 +3294,8 @@ int dw_mci_probe(struct dw_mci *host)
>  	spin_lock_init(&host->irq_lock);
>  	INIT_LIST_HEAD(&host->queue);
>  
> +	dw_mci_init_fault(host);
> +
>  	/*
>  	 * Get the host data width - this assumes that HCON has been set with
>  	 * the correct values.
> diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
> index da5923a92e60..ce05d81477d9 100644
> --- a/drivers/mmc/host/dw_mmc.h
> +++ b/drivers/mmc/host/dw_mmc.h
> @@ -14,6 +14,8 @@
>  #include <linux/mmc/core.h>
>  #include <linux/dmaengine.h>
>  #include <linux/reset.h>
> +#include <linux/fault-inject.h>
> +#include <linux/hrtimer.h>
>  #include <linux/interrupt.h>
>  
>  enum dw_mci_state {
> @@ -230,6 +232,11 @@ struct dw_mci {
>  	struct timer_list       cmd11_timer;
>  	struct timer_list       cto_timer;
>  	struct timer_list       dto_timer;
> +
> +#ifdef CONFIG_FAULT_INJECTION
> +	struct fault_attr	fail_data_crc;
> +	struct hrtimer		fault_timer;
> +#endif
>  };
>  
>  /* DMA ops for Internal/External DMAC interface */
> 


^ permalink raw reply	[flat|nested] 3+ messages in thread

* Re: [PATCH v2] mmc: dw_mmc: Add data CRC error injection
  2021-07-01  8:05 ` [PATCH v2] mmc: dw_mmc: Add data CRC error injection Vincent Whitchurch
  2021-07-25 21:41   ` Jaehoon Chung
@ 2021-08-04 11:34   ` Ulf Hansson
  1 sibling, 0 replies; 3+ messages in thread
From: Ulf Hansson @ 2021-08-04 11:34 UTC (permalink / raw)
  To: Vincent Whitchurch
  Cc: Jaehoon Chung, kernel, linux-mmc, Linux Kernel Mailing List

On Thu, 1 Jul 2021 at 10:05, Vincent Whitchurch
<vincent.whitchurch@axis.com> wrote:
>
> This driver has had problems when handling data errors.  Add fault
> injection support so that the abort handling can be easily triggered and
> regression-tested.  A hrtimer is used to indicate a data CRC error at
> various points during the data transfer.
>
> Note that for the recent problem with hangs in the case of some data CRC
> errors, a udelay(10) inserted at the start of send_stop_abort() greatly
> helped in triggering the error, but I've not included this as part of
> the fault injection support since it seemed too specific.
>
> Signed-off-by: Vincent Whitchurch <vincent.whitchurch@axis.com>

Applied for next, thanks!

Kind regards
Uffe


> ---
>
> Notes:
>     v2: Add missing includes.
>
>  drivers/mmc/host/dw_mmc.c | 73 +++++++++++++++++++++++++++++++++++++++
>  drivers/mmc/host/dw_mmc.h |  7 ++++
>  2 files changed, 80 insertions(+)
>
> diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
> index d333130d1531..dbbb94e6ff4b 100644
> --- a/drivers/mmc/host/dw_mmc.c
> +++ b/drivers/mmc/host/dw_mmc.c
> @@ -17,9 +17,11 @@
>  #include <linux/interrupt.h>
>  #include <linux/iopoll.h>
>  #include <linux/ioport.h>
> +#include <linux/ktime.h>
>  #include <linux/module.h>
>  #include <linux/platform_device.h>
>  #include <linux/pm_runtime.h>
> +#include <linux/prandom.h>
>  #include <linux/seq_file.h>
>  #include <linux/slab.h>
>  #include <linux/stat.h>
> @@ -181,6 +183,9 @@ static void dw_mci_init_debugfs(struct dw_mci_slot *slot)
>                            &host->pending_events);
>         debugfs_create_xul("completed_events", S_IRUSR, root,
>                            &host->completed_events);
> +#ifdef CONFIG_FAULT_INJECTION
> +       fault_create_debugfs_attr("fail_data_crc", root, &host->fail_data_crc);
> +#endif
>  }
>  #endif /* defined(CONFIG_DEBUG_FS) */
>
> @@ -1788,6 +1793,68 @@ static const struct mmc_host_ops dw_mci_ops = {
>         .prepare_hs400_tuning   = dw_mci_prepare_hs400_tuning,
>  };
>
> +#ifdef CONFIG_FAULT_INJECTION
> +static enum hrtimer_restart dw_mci_fault_timer(struct hrtimer *t)
> +{
> +       struct dw_mci *host = container_of(t, struct dw_mci, fault_timer);
> +       unsigned long flags;
> +
> +       spin_lock_irqsave(&host->irq_lock, flags);
> +
> +       if (!host->data_status)
> +               host->data_status = SDMMC_INT_DCRC;
> +       set_bit(EVENT_DATA_ERROR, &host->pending_events);
> +       tasklet_schedule(&host->tasklet);
> +
> +       spin_unlock_irqrestore(&host->irq_lock, flags);
> +
> +       return HRTIMER_NORESTART;
> +}
> +
> +static void dw_mci_start_fault_timer(struct dw_mci *host)
> +{
> +       struct mmc_data *data = host->data;
> +
> +       if (!data || data->blocks <= 1)
> +               return;
> +
> +       if (!should_fail(&host->fail_data_crc, 1))
> +               return;
> +
> +       /*
> +        * Try to inject the error at random points during the data transfer.
> +        */
> +       hrtimer_start(&host->fault_timer,
> +                     ms_to_ktime(prandom_u32() % 25),
> +                     HRTIMER_MODE_REL);
> +}
> +
> +static void dw_mci_stop_fault_timer(struct dw_mci *host)
> +{
> +       hrtimer_cancel(&host->fault_timer);
> +}
> +
> +static void dw_mci_init_fault(struct dw_mci *host)
> +{
> +       host->fail_data_crc = (struct fault_attr) FAULT_ATTR_INITIALIZER;
> +
> +       hrtimer_init(&host->fault_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
> +       host->fault_timer.function = dw_mci_fault_timer;
> +}
> +#else
> +static void dw_mci_init_fault(struct dw_mci *host)
> +{
> +}
> +
> +static void dw_mci_start_fault_timer(struct dw_mci *host)
> +{
> +}
> +
> +static void dw_mci_stop_fault_timer(struct dw_mci *host)
> +{
> +}
> +#endif
> +
>  static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq)
>         __releases(&host->lock)
>         __acquires(&host->lock)
> @@ -2102,6 +2169,7 @@ static void dw_mci_tasklet_func(struct tasklet_struct *t)
>                                 break;
>                         }
>
> +                       dw_mci_stop_fault_timer(host);
>                         host->data = NULL;
>                         set_bit(EVENT_DATA_COMPLETE, &host->completed_events);
>                         err = dw_mci_data_complete(host, data);
> @@ -2151,6 +2219,7 @@ static void dw_mci_tasklet_func(struct tasklet_struct *t)
>                         if (mrq->cmd->error && mrq->data)
>                                 dw_mci_reset(host);
>
> +                       dw_mci_stop_fault_timer(host);
>                         host->cmd = NULL;
>                         host->data = NULL;
>
> @@ -2600,6 +2669,8 @@ static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status)
>
>         set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
>         tasklet_schedule(&host->tasklet);
> +
> +       dw_mci_start_fault_timer(host);
>  }
>
>  static void dw_mci_handle_cd(struct dw_mci *host)
> @@ -3223,6 +3294,8 @@ int dw_mci_probe(struct dw_mci *host)
>         spin_lock_init(&host->irq_lock);
>         INIT_LIST_HEAD(&host->queue);
>
> +       dw_mci_init_fault(host);
> +
>         /*
>          * Get the host data width - this assumes that HCON has been set with
>          * the correct values.
> diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
> index da5923a92e60..ce05d81477d9 100644
> --- a/drivers/mmc/host/dw_mmc.h
> +++ b/drivers/mmc/host/dw_mmc.h
> @@ -14,6 +14,8 @@
>  #include <linux/mmc/core.h>
>  #include <linux/dmaengine.h>
>  #include <linux/reset.h>
> +#include <linux/fault-inject.h>
> +#include <linux/hrtimer.h>
>  #include <linux/interrupt.h>
>
>  enum dw_mci_state {
> @@ -230,6 +232,11 @@ struct dw_mci {
>         struct timer_list       cmd11_timer;
>         struct timer_list       cto_timer;
>         struct timer_list       dto_timer;
> +
> +#ifdef CONFIG_FAULT_INJECTION
> +       struct fault_attr       fail_data_crc;
> +       struct hrtimer          fault_timer;
> +#endif
>  };
>
>  /* DMA ops for Internal/External DMAC interface */
> --
> 2.28.0
>

^ permalink raw reply	[flat|nested] 3+ messages in thread

end of thread, other threads:[~2021-08-04 11:35 UTC | newest]

Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
     [not found] <CGME20210701080542epcas1p1ba3235a77ae2ddccc1d1cb40df2e51de@epcas1p1.samsung.com>
2021-07-01  8:05 ` [PATCH v2] mmc: dw_mmc: Add data CRC error injection Vincent Whitchurch
2021-07-25 21:41   ` Jaehoon Chung
2021-08-04 11:34   ` Ulf Hansson

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).