Linux soft lockup分析
关键词：watchdog、soft lockup、percpu thread、lockdep等。
最近遇到了一个soft lockup问题，打印类似于[ 80.002856] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [poweroff:965]“。
这是lockup检测机制起作用，lockup检测机制包括soft lockup detector和hard lockup detector。
借机分析下soft lockup在什么情况下导致机制和机制？soft watchdog异常、对watchdog配置，如何定位异常点。
这里跳过hard lockup detector的分析。
1. soft lockup机制分析
lockup_detector_init首先获得函数sample_period以及watchdog_cpumask，然后根据情况创建线程，启动喂狗程序；创建hrtimer启动看门狗。
然后有两个关键点，一个是创建内核线程API以及struct smp_hotplug_thread结构体。
void __init lockup_detector_init(void) { /* 获取变量sample_period，为watchdog_thresh*2/5，这是后面设置的4s的hrtime中断。 */ set_sample_period(); ... cpumask_copy(&watchdog_cpumask, cpu_possible_mask); if (watchdog_enabled) watchdog_enable_all_cpus(); } static int watchdog_enable_all_cpus(void) { int err = 0; if (!watchdog_running) { /* 如果当前watchdog_running如果没有再次运行，那么对为了每一个CPU创建一个watchdog/x线程， * 每隔一段线程sample_period喂狗一次。watchdog_threads时watchdog/x线程的主要输入参数， * watchdog_cpumask规定了为什么CPU创建线程。 */ err = smpboot_register_percpu_thread_cpumask(&watchdog_threads, &watchdog_cpumask); if (err) pr_err("Failed to create watchdog threads, disabled\n"); else watchdog_running = 1; } else { err = update_watchdog_all_cpus(); if (err) { watchdog_disable_all_cpus(); pr_err("Failed to update lockup detectors, disabled\n"); } } if (err) watchdog_enabled = 0; return err; } static int update_watchdog_all_cpus(void) { int ret; ret = watchdog_park_threads(); if (ret) return ret; watchdog_unpark_threads(); return 0; } static int watchdog_park_threads(void) { int cpu, ret = 0; atomic_set(&watchdog_park_in_progress, 1); for_each_watchdog_cpu(cpu) { /* 设置struct kthread->flags的KTHREAD_SHOULD_PARK位 * 在watchdog/x调用线程unpark成员函数进行处理 */ ret = kthread_park(per_cpu(softlockup_watchdog, cpu)); if (ret) break; } atomic_set(&watchdog_park_in_progress, 0); return ret; } 1.1 watchdog_threads结构体介绍
- 介绍如何创建watchdog/x有必要在线程之前介绍一些struct smp_hotplug_thread线程。
struct smp_hotplug_thread { struct task_struct __percpu **store; /*存放percpu strcut task_strcut指针的指针*/ struct list_head list; int (*thread_should_run)(unsigned int cpu); /* 检查是否应运行watchdog/x线程 */ void (*thread_fn)(unsigned int cpu); /* watchdog/x线程主函数 */ void (*create)(unsigned int cpu); void (*setup)(unsigned int cpu); /* 运行watchdog/x线程前的准备工作 */ void (*cleanup)(unsigned int cpu, bool online); /* 在退出watchdog/x线程之后的清楚工作 */ void (*park)(unsigned int cpu); /*当CPU offline时，需要临时停止*/ void (*unpark)(unsigned int cpu); /*当CPU变成online时，准备工作*/ cpumask_var_t cpumask; /* 允许哪些CPU online */ bool selfparking; const char *thread_comm; /* watchdog/x线程名称 */ }; - watchdog_threads是soft lockup基于此创建监控线程的实体 watchdog/x线程。
static struct smp_hotplug_thread watchdog_threads = { .store = &softlockup_watchdog, .thread_should_run = watchdog_should_run, .thread_fn = watchdog, .thread_comm = "watchdog/%u", .setup = watchdog_enable, .cleanup = watchdog_cleanup, .park = watchdog_disable, .unpark = watchdog_enable, }; static void watchdog_enable(unsigned int cpu) { struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer); /* kick off the timer for the hardlockup detector */ hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); /* 创建一个hrtimer，超时函数为watchdog_timer_fn， * 会检查的watchdog_touch_ts变量是否超过20秒未更新。 * 如果是，则有soft lockup */ hrtimer->function = watchdog_timer_fn; /* Enable the perf event */ watchdog_nmi_enable(cpu); /* done here because hrtimer_start can only pin to smp_processor_id() */ /* 启动一个sample_period(4秒)的hrtimer，HRTIMER_MODE_REL_PINNED表示此hrtimer和当前CPU绑定 /
hrtimer_start(hrtimer, ns_to_ktime(sample_period),
HRTIMER_MODE_REL_PINNED);
/* initialize timestamp */
/* 设置当前线程为实时FIFO，并且优先级为实时99.这个优先级表示高于所有的非实时线程，但是实时优先级最低的*/
watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1);
/* 更新watchdog_touch_ts变量，相当于喂狗操作。 */
__touch_watchdog();
}
static void watchdog_set_prio(unsigned int policy, unsigned int prio)
{
struct sched_param param = { .sched_priority = prio };
sched_setscheduler(current, policy, ¶m);
}
/* Commands for resetting the watchdog */
static void __touch_watchdog(void)
{
/* 喂狗的操作就是更新watchdog_touch_ts变量，就是当前时间戳更新到watchdog_touch_ts中*/
__this_cpu_write(watchdog_touch_ts, get_timestamp());
}
static int watchdog_should_run(unsigned int cpu)
{
/* hrtimer_interrupts记录了产生hrtimer的次数；
* 在watchdog()中，将hrtimer_interrupts赋给soft_lockup_hrtimer_cnt。两者相等表示没有hrtimer产生，
* 不需要运行watchdog/x线程；相反不等，则需要watchdog/x线程运行。
*/
return __this_cpu_read(hrtimer_interrupts) !=
__this_cpu_read(soft_lockup_hrtimer_cnt);
}
static void watchdog(unsigned int cpu)
{
/* 更新soft_lockup_hrtimer_cnt，在watch_should_run()中就返回false，
* 表示线程不需要运行，即不需要喂狗。
*/
__this_cpu_write(soft_lockup_hrtimer_cnt,
__this_cpu_read(hrtimer_interrupts));
/*虽然就是一句话，但是却很重要的喂狗操作。*/
__touch_watchdog();
if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED))
watchdog_nmi_disable(cpu);
}
1.2 创建喂狗线程watchdog/x
在分析了[watchdog_threads]之后，再来看看如何创建watchdog/x线程。
int smpboot_register_percpu_thread_cpumask(struct smp_hotplug_thread *plug_thread,
const struct cpumask *cpumask)
{
unsigned int cpu;
int ret = 0;
if (!alloc_cpumask_var(&plug_thread->cpumask, GFP_KERNEL))
return -ENOMEM;
cpumask_copy(plug_thread->cpumask, cpumask);
get_online_cpus();
mutex_lock(&smpboot_threads_lock);
for_each_online_cpu(cpu) {------------------------------------------------遍历所有online CPU，为每个CPU创建一个percpu的watchdog/x线程。
ret = __smpboot_create_thread(plug_thread, cpu);
if (ret) {
smpboot_destroy_threads(plug_thread);-----------------------------创建失败则释放相关资源。
free_cpumask_var(plug_thread->cpumask);
goto out;
}
if (cpumask_test_cpu(cpu, cpumask))
smpboot_unpark_thread(plug_thread, cpu);--------------------------如果当前CPU不在cpumask中，则清空KTHREAD_SHOULD_PARK，进而调用watchdog_therads的umpark成员函数。
}
list_add(&plug_thread->list, &hotplug_threads);
out:
mutex_unlock(&smpboot_threads_lock);
put_online_cpus();
return ret;
}
static int
__smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
{
struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
struct smpboot_thread_data *td;
if (tsk)
return 0;
td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu));
if (!td)
return -ENOMEM;
td->cpu = cpu;
td->ht = ht;
tsk =kthread_create_on_cpu(smpboot_thread_fn, td, cpu,
ht->thread_comm);-----------------------------------------在指定CPU上创建watchdog/x线程，处理函数为smpboot_thread_fn()。
if (IS_ERR(tsk)) {
kfree(td);
return PTR_ERR(tsk);
}
/*
* Park the thread so that it could start right on the CPU
* when it is available.
*/
kthread_park(tsk);--------------------------------------------------------在CPU上立即启动watchdog/x线程。
get_task_struct(tsk);-----------------------------------------------------增加对线程的引用计数。
*per_cpu_ptr(ht->store, cpu) = tsk;---------------------------------------store存放线程结构体指针的指针。
if (ht->create) {
if (!wait_task_inactive(tsk, TASK_PARKED))
WARN_ON(1);
else
ht->create(cpu);
}
return 0;
}
static int smpboot_thread_fn(void *data)
{
struct smpboot_thread_data *td = data;
struct smp_hotplug_thread *ht = td->ht;
while (1) {
set_current_state(TASK_INTERRUPTIBLE);
preempt_disable();
if (kthread_should_stop()) {----------------------------------------如果可以终止线程，调用cleanup，退出线程。
__set_current_state(TASK_RUNNING);
preempt_enable();
/* cleanup must mirror setup */
if (ht->cleanup && td->status != HP_THREAD_NONE)
ht->cleanup(td->cpu, cpu_online(td->cpu));
kfree(td);
return 0;
}
if (kthread_should_park()) {----------------------------------------如果KTHREAD_SHOULD_PARK置位，调用park()暂停进程执行。
__set_current_state(TASK_RUNNING);
preempt_enable();
if (ht->park && td->status == HP_THREAD_ACTIVE) {
BUG_ON(td->cpu != smp_processor_id());
ht->park(td->cpu);
td->status = HP_THREAD_PARKED;
}
kthread_parkme();
/* We might have been woken for stop */
continue;
}
BUG_ON(td->cpu != smp_processor_id());
/* Check for state change setup */
switch (td->status) {
case HP_THREAD_NONE:-----------------------------------------------相当于第一次运行，调用setup()进行初始化操作。
__set_current_state(TASK_RUNNING);
preempt_enable();
if (ht->setup)
ht->setup(td->cpu);
td->status = HP_THREAD_ACTIVE;
continue;
case HP_THREAD_PARKED:---------------------------------------------从parked状态恢复。
__set_current_state(TASK_RUNNING);
preempt_enable();
if (ht->unpark)
ht->unpark(td->cpu);
td->status = HP_THREAD_ACTIVE;
continue;
}
if (!ht->thread_should_run(td->cpu)) {-----------------------------如果不需要进程运行，schedule()主动放弃CPU给其他线程使用。
preempt_enable_no_resched();
schedule();
} else {
__set_current_state(TASK_RUNNING);
preempt_enable();
ht->thread_fn(td->cpu);----------------------------------------调用struct smpboot_thread_fn->thread_fn及watchdog()，进行喂狗操作。
}
}
}
void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread)----将创建的内核线程移除操作。
{
get_online_cpus();
mutex_lock(&smpboot_threads_lock);
list_del(&plug_thread->list);
smpboot_destroy_threads(plug_thread);
mutex_unlock(&smpboot_threads_lock);
put_online_cpus();
free_cpumask_var(plug_thread->cpumask);
}
static void smpboot_destroy_threads(struct smp_hotplug_thread *ht)
{
unsigned int cpu;
/* We need to destroy also the parked threads of offline cpus */
for_each_possible_cpu(cpu) {
struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
if (tsk) {
kthread_stop(tsk);
put_task_struct(tsk);
*per_cpu_ptr(ht->store, cpu) = NULL;
}
}
}
1.3 hrtimer看门狗
在分析了喂狗线程watchdog/x之后，再来分析看门狗是如何实现的？
看门狗是通过启动一个周期为4秒的hrtimer来实现的，这个hrtimer和CPU绑定，使用的变量都是percpu的。确保每个CPU之间不相互干扰。
每次hrtimer超时，都会唤醒watchdog/x线程，并进行一次喂狗操作。
因为hrtimer超时函数在软中断中调用，在中断产生后会比线程优先得到执行。
所以在watchdog/x线程没有得到执行的情况下，通过is_softlockup来判断看门狗是否超过20秒没有得到喂狗。
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) { unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts); struct pt_regs *regs = get_irq_regs(); int duration; int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace; if (atomic_read(&watchdog_park_in_progress) != 0) return HRTIMER_NORESTART; /* kick the hardlockup detector */ /*每产生一次中断，hrtimer_interrupts计数加1.hrtimer_interrupts记录了产生hrtimer的次数。*/ watchdog_interrupt_count(); /* kick the softlockup detector */ /*唤醒watchdog/x线程，进行喂狗操作,如果此时的调度被关闭，被喂狗线程得不到执行，可手动preempt_disbale()关闭调度测试softlock*/ wake_up_process(__this_cpu_read(softlockup_watchdog)); /* .. and repeat 新设置超时点，形成周期性时钟。*/ hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period)); ... /* 返回非0表示，看门狗超时。 */ duration = is_softlockup(touch_ts); if (unlikely(duration)) { if (kvm_check_and_clear_guest_paused()) return HRTIMER_RESTART; /* only warn once */ if (__this_cpu_read(soft_watchdog_warn) == true) { if (__this_cpu_read(softlockup_task_ptr_saved) != current) { __this_cpu_write(soft_watchdog_warn, false); __touch_watchdog(); } return HRTIMER_RESTART; } if (softlockup_all_cpu_backtrace) { if (test_and_set_bit(0, &soft_lockup_nmi_warn)) { /* Someone else will report us. Let's give up */ __this_cpu_write(soft_watchdog_warn, true); return HRTIMER_RESTART; } } /* 打印哪个CPU被卡死duration秒，以及死在哪个进程。*/ pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n", smp_processor_id(), duration, current->comm, task_pid_nr(current)); __this_cpu_write(softlockup_task_ptr_saved, current); print_modules(); /* 显示开关中断、软中断信息，禁止中断和软中断也是造成soft lockup的一个原因。 */ print_irqtrace_events(current); /* 有寄存器显示寄存器信息，同时显示栈信息。 */ if (regs) show_regs(regs); else dump_stack(); if (softlockup_all_cpu_backtrace) { trigger_allbutself_cpu_backtrace(); clear_bit(0, &soft_lockup_nmi_warn); /* Barrier to sync with other cpus */ smp_mb__after_atomic(); } add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); /* 如果定义softlockup_panic则进入panic()。 */ if (softlockup_panic) panic("softlockup: hung tasks"); __this_cpu_write(soft_watchdog_warn, true); } else __this_cpu_write(soft_watchdog_warn, false); return HRTIMER_RESTART; }
static void watchdog_interrupt_count(void) { __this_cpu_inc(hrtimer_interrupts); }
static int is_softlockup(unsigned long touch_ts)
{
unsigned long now = get_timestamp();
if ((watchdog_enabled & SOFT_WATCHDOG_ENABLED) && watchdog_thresh){
/* Warn about unreasonable delays. */
if (time_after(now, touch_ts + get_softlockup_thresh()))
return now - touch_ts;
}
return 0;
}
2. 对watchdog的设置
对watchdog行为的设置有两个途径：通过命令行传入参数和通过proc设置。
2.1 通过命令行设置
static int __init softlockup_panic_setup(char *str)
{
softlockup_panic = simple_strtoul(str, NULL, 0);
return 1;
}
__setup("softlockup_panic=", softlockup_panic_setup);
static int __init nowatchdog_setup(char *str)
{
watchdog_enabled = 0;
return 1;
}
__setup("nowatchdog", nowatchdog_setup);
static int __init nosoftlockup_setup(char *str)
{
watchdog_enabled &= ~SOFT_WATCHDOG_ENABLED;
return 1;
}
__setup("nosoftlockup", nosoftlockup_setup);
#ifdef CONFIG_SMP
static int __init softlockup_all_cpu_backtrace_setup(char *str)
{
sysctl_softlockup_all_cpu_backtrace =
!!simple_strtol(str, NULL, 0);
return 1;
}
__setup("softlockup_all_cpu_backtrace=", softlockup_all_cpu_backtrace_setup);
static int __init hardlockup_all_cpu_backtrace_setup(char *str)
{
sysctl_hardlockup_all_cpu_backtrace =
!!simple_strtol(str, NULL, 0);
return 1;
}
__setup("hardlockup_all_cpu_backtrace=", hardlockup_all_cpu_backtrace_setup);
#endif
2.2 通过sysfs节点调节watchdog
watchdog相关的配置还可以通过proc文件系统进行配置。
/proc/sys/kernel/nmi_watchdog-------------------------hard lockup开关，proc_nmi_watchdog()。
/proc/sys/kernel/soft_watchdog------------------------soft lockup开关，proc_soft_watchdog()。
/proc/sys/kernel/watchdog-----------------------------watchdog总开关，proc_watchdog()。
/proc/sys/kernel/watchdog_cpumask---------------------watchdog cpumaks，proc_watchdog_cpumask()。
/proc/sys/kernel/watchdog_thresh----------------------watchdog超时阈值设置，proc_watchdog_thresh()。
3. 定位soft lockup异常
引起soft lockup的原因一般是死循环或者死锁， 死循环可以通过栈回溯找到问题点；死锁问题需要打开内核的lockdep功能。
打开内核的lockdep功能可以参考《Linux死锁检测-Lockdep》。
下面看一个关闭调度并while(1)引起的soft lockup异常分析：
[ 664.049439] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [init:980]
[ 664.057346] Modules linked in: [last unloaded: videobuf2_dma_contig]
[ 664.064495] CPU: 0 PID: 980 Comm: init Not tainted 4.9.118 #118
[ 664.071135] Hardware name: sun8iw19
[ 664.075050] task: ddc27a40 task.stack: ddc6e000
[ 664.080143] PC is at axp2101_shutdown+0x24/0x30 /* 这里PC指针直接打印出了函数名 */
[ 664.085229] LR is at axp2101_shutdown+0x24/0x30
[ 664.090315] pc : [<c0324c24>] lr : [<c0324c24>] psr: 60000013
[ 664.090315] sp : ddc6fe58 ip : c0793238 fp : 00000000
[ 664.103200] r10: 00000000 r9 : fee1dead r8 : df23d844
[ 664.109063] r7 : ddc6e000 r6 : df23d020 r5 : df23d810 r4 : df23d81c
[ 664.116388] r3 : 00000000 r2 : 00000001 r1 : 00000002 r0 : 00000023
[ 664.123715] Flags: nZCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment none
[ 664.131723] Control: 10c5387d Table: 5dc50059 DAC: 00000051
[ 664.138168]
[ 664.138168] PC: 0xc0324ba4:
[ 664.142959] 4ba4 ebfd2738 e3a00000 e8bd8010 c0686844 e92d4010 e5904064 e594327c e3530000
[ 664.152195] 4bc4 1a000002 e594102c e1a00004 ebfffd50 e2840f8e ebf441d4 e1a00004 e3a01000
[ 664.161430] 4be4 ebffff5d e2842fa1 e3a01081 e5940224 ebfd229e e3a00000 e8bd8010 e92d4010
[ 664.170664] 4c04 e59f001c ebf54f6c e59f0018 ebf54f6a e3a00001 ebf462ce e59f0004 ebf54f66
[ 664.179899] 4c24 eafffffe c05a3aaf c05e013c e92d47f0 e24dd028 e1a04000 e28d2010 e3a010a2
[ 664.189133] 4c44 e5100014 e3a03000 e1cd30be ebfd2287 e59d3010 e3130010 0a000019 e28d2020
[ 664.198366] 4c64 e3a010b0 e5140014 ebfd2280 e59d3020 e353002e 1a000012 e28d2010 e3a010b2
[ 664.207601] 4c84 e5140014 ebfd2279 e3a010b3 e5140014 e28d2014 ebfd2275 e59d2010 e59d3014
[ 664.216837]
[ 664.216837] LR: 0xc0324ba4:
[ 664.221627] 4ba4 ebfd2738 e3a00000 e8bd8010 c0686844 e92d4010 e5904064 e594327c e3530000
[ 664.230861] 4bc4 1a000002 e594102c e1a00004 ebfffd50 e2840f8e ebf441d4 e1a00004 e3a01000
[ 664.240095] 4be4 ebffff5d e2842fa1 e3a01081 e5940224 ebfd229e e3a00000 e8bd8010 e92d4010
[ 664.249329] 4c04 e59f001c ebf54f6c e59f0018 ebf54f6a e3a00001 ebf462ce e59f0004 ebf54f66
[ 664.258563] 4c24 eafffffe c05a3aaf c05e013c e92d47f0 e24dd028 e1a04000 e28d2010 e3a010a2
[ 664.267789] 4c44 e5100014 e3a03000 e1cd30be ebfd2287 e59d3010 e3130010 0a000019 e28d2020
[ 664.277023] 4c64 e3a010b0 e5140014 ebfd2280 e59d3020 e353002e 1a000012 e28d2010 e3a010b2
[ 664.286257] 4c84 e5140014 ebfd2279 e3a010b3 e5140014 e28d2014 ebfd2275 e59d2010 e59d3014
[ 664.295492]
[ 664.295492] SP: 0xddc6fdd8:
[ 664.300282] fdd8 e0803000 ddc6fe08 df23d844 ddc6e000 00000000 c0009494 c0324c24 60000013
[ 664.309516] fdf8 ffffffff ddc6fe3c df23d844 c001348c 00000023 00000002 00000001 00000000
[ 664.318749] fe18 df23d81c df23d810 df23d020 ddc6e000 df23d844 fee1dead 00000000 00000000
[ 664.327982] fe38 c0793238 ddc6fe58 c0324c24 c0324c24 60000013 ffffffff 00000051 bf000000
[ 664.337207] fe58 df23d81c c0257ecc 4321fedc 4321fedc c068680c c068d010 00000000 c003ba24
[ 664.346441] fe78 4321fedc c003bc08 ddc6fe8c c003eb48 ffffe000 c0021d94 ddc6fe90 de608870
[ 664.355675] fe98 bea61000 00000055 ddc52fa8 de683184 de262038 00000000 ddc6ffb0 ddc6ffb0
[ 664.364908] feb8 00000055 bea61c4c ddc27a40 0000080f de262000 de262038 00000800 c00198fc
[ 664.374143]
[ 664.374143] IP: 0xc07931b8:
[ 664.378933] 31b8 00000001 00000000 00000000 00000000 00000000 c07931cc c07931cc 00000000
[ 664.388166] 31d8 00000000 c06938f0 00000000 00000001 00000001 00000000 c051e1d8 c07931f4
[ 664.397399] 31f8 c07931f4 00400000 00000001 00000000 00000000 00000000 00000000 00000000
[ 664.406631] 3218 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.415863] 3238 00000001 00000002 00000000 00000000 00000000 00000000 00000000 df2fff00
[ 664.425094] 3258 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.434326] 3278 00000001 00000004 00000000 00000000 00000000 00000000 00000000 df2d0380
[ 664.443558] 3298 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.452791]
[ 664.452791] R4: 0xdf23d79c:
[ 664.457581] d79c 6f74616c 00000072 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.466814] d7bc 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.476045] d7dc 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.485277] d7fc 00000000 df23d990 00000000 00000000 00000000 df23d020 df20ef00 df22fa80
[ 664.494509] d81c df23d81c df23d81c df23d028 df02e140 c0755c04 df224550 00000008 00000007
[ 664.503742] d83c 00000000 c0756bdc 00000000 df23d848 df23d848 ddc27a40 df23d844 c0755ed0
[ 664.512976] d85c c076c09c 00000000 df3fe010 00000000 00000080 df23da70 df2730b0 7fffffff
[ 664.522209] d87c df23d87c df23d87c 00000000 00000000 00000000 00000000 00000000 c0262db4
[ 664.531443]
[ 664.531443] R5: 0xdf23d790:
[ 664.536233] d790 32707861 2d313031 75676572 6f74616c 00000072 00000000 00000000 00000000
[ 664.545467] d7b0 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.554697] d7d0 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.563928] d7f0 00000000 00000000 00000000 00000000 df23d990 00000000 00000000 00000000
[ 664.573159] d810 df23d020 df20ef00 df22fa80 df23d81c df23d81c df23d028 df02e140 c0755c04
[ 664.582393] d830 df224550 00000008 00000007 00000000 c0756bdc 00000000 df23d848 df23d848
[ 664.591626] d850 ddc27a40 df23d844 c0755ed0 c076c09c 00000000 df3fe010 00000000 00000080
[ 664.600860] d870 df23da70 df2730b0 7fffffff df23d87c df23d87c 00000000 00000000 00000000
[ 664.610094]
[ 664.610094] R6: 0xdf23cfa0:
[ 664.614885] cfa0 00000001 00000000 df23cf50 c05a129a df23e061 df23e100 df23e0b0 00000000
[ 664.624117] cfc0 4e121040 00000000 c04e23d8 00000000 00001000 00000000 00000000 00000000
[ 664.633349] cfe0 c068fcd8 81240052 00002178 00000000 00000000 00000000 00000000 00000000
[ 664.642581] d000 00340080 32707861 00313031 00000000 00000000 00000000 df215010 00000000
[ 664.651813] d020 df215048 df255c00 df22b140 df23d41c df215054 df215050 df02e140 c0755c04
[ 664.661047] d040 df23c910 00000037 00000007 00000000 c076abd0 00000000 df23d058 df23d058
[ 664.670280] d060 ddc27a40 df23d054 c076ac00 c07587a4 00000000 df22be90 00000000 00000081
[ 664.679512] d080 df23d470 df2150a8 7fffffff df23d08c df23d08c df255e00 00000000 00000000
[ 664.688738]
[ 664.688738] R7: 0xddc6df80:
[ 664.693529] df80 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.702761] dfa0 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.711992] dfc0 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.721223] dfe0 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.730454] e000 00000002 00010001 00000000 ddc27a40 00000000 00000000 ddc27a40 c068d360
[ 664.739686] e020 c0688c78 de262000 00000000 fee1dead 00000000 ddc6fda4 ddc6fd80 c04d63ec
[ 664.748920] e040 00000000 00000000 00000000 00000000 00000000 01010000 00000000 b6f8f540
[ 664.758151] e060 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.767383]
[ 664.767383] R8: 0xdf23d7c4:
[ 664.772173] d7c4 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[ 664.781404] d7e4 00000000 00000000 00000000 00000000 00000000 00000000 00000000 df23d990
[ 664.790635] d804 00000000 00000000 00000000 df23d020 df20ef00 df22fa80 df23d81c df23d81c
[ 664.799867] d824 df23d028 df02e140 c0755c04 df224550 00000008 00000007 00000000 c0756bdc
[ 664.809092] d844 00000000 df23d848 df23d848 ddc27a40 df23d844 c0755ed0 c076c09c 00000000
[ 664.818325] d864 df3fe010 00000000 00000080 df23da70 df2730b0 7fffffff df23d87c df23d87c
[ 664.827557] d884 00000000 00000000 00000000 00000000 00000000 c0262db4 df23d810 00000000
[ 664.836789] d8a4 00000000 ffffffe0 df23d8ac df23d8ac c0262e4c df23d8b8 df23d8b8 00000000
[ 664.846015]
[ 664.846015] R9: 0xfee1de2d:
[ 664.850805] de2c ******** ******** ******** ******** ******** ******** ******** ********
[ 664.860046] de4c ******** ******** ******** ******** ******** ******** ******** ********
[ 664.869279] de6c ******** ******** ******** ******** ******** ******** ******** ********
[ 664.878511] de8c ******** ******** ******** ******** ******** ******** ******** ********
[ 664.887743] deac ******** ******** ******** ******** ******** ******** ******** ********
[ 664.896976] decc ******** ******** ******** ******** ******** ******** ******** ********
[ 664.906208] deec ******** ******** ******** ******** ******** ******** ******** ********
[ 664.915441] df0c ******** ******** ******** ******** ******** ******** ******** ********
[ 664.924673] df2c ******** ******** ******** ******** ******** ******** ******** ********
[ 664.933913] CPU: 0 PID: 980 Comm: init Not tainted 4.9.118 #118
[ 664.940553] Hardware name: sun8iw19
[ 664.944494] [<c00158d8>] (unwind_backtrace) from [<c0012ba4>] (show_stack+0x10/0x14)
[ 664.953203] [<c0012ba4>] (show_stack) from [<c0075484>] (watchdog_timer_fn+0x1ec/0x27c)
[ 664.962203] [<c0075484>] (watchdog_timer_fn) from [<c005dd74>] (__hrtimer_run_queues.constprop.4+0xdc/0x184)
[ 664.973247] [<c005dd74>] (__hrtimer_run_queues.constprop.4) from [<c005dfdc>] (hrtimer_interrupt+0xa8/0x230)
[ 664.984290] [<c005dfdc>] (hrtimer_interrupt) from [<c035708c>] (arch_timer_handler_phys+0x28/0x30)
[ 664.994360] [<c035708c>] (arch_timer_handler_phys) from [<c00520b0>] (handle_percpu_devid_irq+0x70/0x118)
[ 665.005113] [<c00520b0>] (handle_percpu_devid_irq) from [<c004de0c>] (generic_handle_irq+0x18/0x28)
[ 665.015279] [<c004de0c>] (generic_handle_irq) from [<c004e394>] (__handle_domain_irq+0xb0/0xcc)
[ 665.025053] [<c004e394>] (__handle_domain_irq) from [<c0009494>] (gic_handle_irq+0x44/0x64)
[ 665.034433] [<c0009494>] (gic_handle_irq) from [<c001348c>] (__irq_svc+0x6c/0xa8)
[ 665.042830] Exception stack(0xddc6fe08 to 0xddc6fe50)
[ 665.048504] fe00: 00000023 00000002 00000001 00000000 df23d81c df23d810
[ 665.057689] fe20: df23d020 ddc6e000 df23d844 fee1dead 00000000 00000000 c0793238 ddc6fe58
[ 665.066868] fe40: c0324c24 c0324c24 60000013 ffffffff
[ 665.072547] [<c001348c>] (__irq_svc) from [<c0324c24>] (axp2101_shutdown+0x24/0x30)
[ 665.081154] [<c0324c24>] (axp2101_shutdown) from [<c0257ecc>] (device_shutdown+0x1b0/0x200)
[ 665.090542] [<c0257ecc>] (device_shutdown) from [<c003ba24>] (kernel_power_off+0x2c/0x70)
[ 665.099732] [<c003ba24>] (kernel_power_off) from [<c003bc08>] (SyS_reboot+0x158/0x1d4)
[ 665.108628] [<c003bc08>] (SyS_reboot) from [<c000ed80>] (ret_fast_syscall+0x0/0x48)
[ 665.117217] Kernel panic - not syncing: softlockup: hung tasks
[ 665.123766] CPU: 0 PID: 980 Comm: init Tainted: G L 4.9.118 #118
[ 665.131770] Hardware name: sun8iw19
[ 665.135696] [<c00158d8>] (unwind_backtrace) from [<c0012ba4>] (show_stack+0x10/0x14)
[ 665.144397] [<c0012ba4>] (show_stack) from [<c0078798>] (panic+0xac/0x278)
[ 665.152123] [<c0078798>] (panic) from [<c00754a4>] (watchdog_timer_fn+0x20c/0x27c)
[ 665.160631] [<c00754a4>] (watchdog_timer_fn) from [<c005dd74>] (__hrtimer_run_queues.constprop.4+0xdc/0x184)
[ 665.171671] [<c005dd74>] (__hrtimer_run_queues.constprop.4) from [<c005dfdc>] (hrtimer_interrupt+0xa8/0x230)
[ 665.182713] [<c005dfdc>] (hrtimer_interrupt) from [<c035708c>] (arch_timer_handler_phys+0x28/0x30)
[ 665.192778] [<c035708c>] (arch_timer_handler_phys) from [<c00520b0>] (handle_percpu_devid_irq+0x70/0x118)
[ 665.203527] [<c00520b0>] (handle_percpu_devid_irq) from [<c004de0c>] (generic_handle_irq+0x18/0x28)
[ 665.213691] [<c004de0c>] (generic_handle_irq) from [<c004e394>] (__handle_domain_irq+0xb0/0xcc)
[ 665.223463] [<c004e394>] (__handle_domain_irq) from [<c0009494>] (gic_handle_irq+0x44/0x64)
[ 665.232842] [<c0009494>] (gic_handle_irq) from [<c001348c>] (__irq_svc+0x6c/0xa8)
[ 665.241239] Exception stack(0xddc6fe08 to 0xddc6fe50)
[ 665.246911] fe00: 00000023 00000002 00000001 00000000 df23d81c df23d810
[ 665.256094] fe20: df23d020 ddc6e000 df23d844 fee1dead 00000000 00000000 c0793238 ddc6fe58
[ 665.265274] fe40: c0324c24 c0324c24 60000013 ffffffff
[ 665.270952] [<c001348c>] (__irq_svc) from [<c0324c24>] (axp2101_shutdown+0x24/0x30)
[ 665.279555] [<c0324c24>] (axp2101_shutdown) from [<c0257ecc>] (device_shutdown+0x1b0/0x200)
[ 665.288939] [<c0257ecc>] (device_shutdown) from [<c003ba24>] (kernel_power_off+0x2c/0x70)
[ 665.298128] [<c003ba24>] (kernel_power_off) from [<c003bc08>] (SyS_reboot+0x158/0x1d4)
[ 665.307024] [<c003bc08>] (SyS_reboot) from [<c000ed80>] (ret_fast_syscall+0x0/0x48)
[ 665.315636] ---[ end Kernel panic - not syncing: softlockup: hung tasks