Hey,
Our mysql 5.5 database is hosted in a ec2 instance and during one of our daily backup runs, we noticed the following in /var/log/messages, shortly we lost access to the box,so could not dive deeper. Any help will be immensely appreciated!
cac7>] handle_percpu_irq+0x47/0x60
[63596.468086] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[63596.468086] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[63596.468086] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[63596.468086] <EOI> [<ffffffff81232669>] ? radix_tree_lookup_slot+0x9/0x10
[63596.468086] [<ffffffff8123266e>] ? radix_tree_lookup_slot+0xe/0x10
[63596.468086] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[63596.468086] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[63596.468086] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[63596.468086] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[63596.468086] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[63596.468086] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[63596.468086] [<ffffffff81159670>] vfs_read+0xb0/0x180
[63596.468086] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[63596.468086] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] INFO: rcu_sched self-detected stall on CPU { 0} (t=1140075 jiffies)
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] Call Trace:
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff810c984b>] ? handle_irq_event_percpu+0xbb/0x220
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] <EOI> [<ffffffff81232669>] ? radix_tree_lookup_slot+0x9/0x10
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff8123266e>] ? radix_tree_lookup_slot+0xe/0x10
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[68636.804086] INFO: rcu_sched self-detected stall on CPU { 0} (t=2400159 jiffies)
[68636.804086] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62.x86_64 #1
[68636.804086] Call Trace:
[68636.804086] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[68636.804086] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[68636.804086] [<ffffffff81059558>] update_process_times+0x48/0x90
[68636.804086] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[68636.804086] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[68636.804086] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[68636.804086] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[68636.804086] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[68636.804086] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[68636.804086] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[68636.804086] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[68636.804086] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[68636.804086] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[68636.804086] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[68636.804086] <EOI> [<ffffffff812325f8>] ? radix_tree_lookup_element+0x28/0x90
[68636.804086] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
[68636.804086] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[68636.804086] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[68636.804086] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[68636.804086] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[68636.804086] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[68636.804086] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[68636.804086] [<ffffffff81159670>] vfs_read+0xb0/0x180
[68636.804086] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[68636.804086] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] INFO: rcu_sched self-detected stall on CPU { 0} (t=2400159 jiffies)
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] Call Trace:
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] <EOI> [<ffffffff812325f8>] ? radix_tree_lookup_element+0x28/0x90
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[74397.188049] INFO: rcu_sched self-detected stall on CPU { 0} (t=3840255 jiffies)
[74397.188049] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
[74397.188049] Call Trace:
[74397.188049] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[74397.188049] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[74397.188049] [<ffffffff81059558>] update_process_times+0x48/0x90
[74397.188049] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[74397.188049] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[74397.188049] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[74397.188049] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[74397.188049] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[74397.188049] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[74397.188049] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[74397.188049] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[74397.188049] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[74397.188049] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[74397.188049] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[74397.188049] <EOI> [<ffffffff812325ff>] ? radix_tree_lookup_element+0x2f/0x90
[74397.188049] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
[74397.188049] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[74397.188049] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[74397.188049] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[74397.188049] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[74397.188049] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[74397.188049] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[74397.188049] [<ffffffff81159670>] vfs_read+0xb0/0x180
[74397.188049] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[74397.188049] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] INFO: rcu_sched self-detected stall on CPU { 0} (t=3840255 jiffies)
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] Call Trace:
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] <EOI> [<ffffffff812325ff>] ? radix_tree_lookup_element+0x2f/0x90
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[74577.200087] INFO: rcu_sched self-detected stall on CPU { 0} (t=3885258 jiffies)
[74577.200096] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
[74577.200096] Call Trace:
[74577.200096] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[74577.200096] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[74577.200096] [<ffffffff81059558>] update_process_times+0x48/0x90
[74577.200096] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[74577.200096] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[74577.200096] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[74577.200096] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[74577.200096] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[74577.200096] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[74577.200096] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[74577.200096] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[74577.200096] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[74577.200096] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[74577.200096] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[74577.200096] <EOI> [<ffffffff812325d5>] ? radix_tree_lookup_element+0x5/0x90
[74577.200096] [<ffffffff8123266e>] ? radix_tree_lookup_slot+0xe/0x10
[74577.200096] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[74577.200096] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[74577.200096] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[74577.200096] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[74577.200096] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[74577.200096] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[74577.200096] [<ffffffff81159670>] vfs_read+0xb0/0x180
[74577.200096] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[74577.200096] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200087] INFO: rcu_sched self-detected stall on CPU { 0} (t=3885258 jiffies)
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] Call Trace:
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] <EOI> [<ffffffff812325d5>] ? radix_tree_lookup_element+0x5/0x90
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff8123266e>] ? radix_tree_lookup_slot+0xe/0x10
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[74757.212089] INFO: rcu_sched self-detected stall on CPU { 0} (t=3930261 jiffies)
[74757.212089] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
[74757.212089] Call Trace:
[74757.212089] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[74757.212089] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[74757.212089] [<ffffffff81059558>] update_process_times+0x48/0x90
[74757.212089] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[74757.212089] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[74757.212089] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[74757.212089] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[74757.212089] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[74757.212089] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[74757.212089] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[74757.212089] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[74757.212089] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[74757.212089] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[74757.212089] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[74757.212089] <EOI> [<ffffffff812325f8>] ? radix_tree_lookup_element+0x28/0x90
[74757.212089] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
[74757.212089] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[74757.212089] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[74757.212089] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[74757.212089] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[74757.212089] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[74757.212089] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[74757.212089] [<ffffffff81159670>] vfs_read+0xb0/0x180
[74757.212089] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[74757.212089] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] INFO: rcu_sched self-detected stall on CPU { 0} (t=3930261 jiffies)
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] Call Trace:
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] <EOI> [<ffffffff812325f8>] ? radix_tree_lookup_element+0x28/0x90
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[74937.224105] INFO: rcu_sched self-detected stall on CPU { 0} (t=3975264 jiffies)
[74937.224108] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
[74937.224108] Call Trace:
[74937.224108] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[74937.224108] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[74937.224108] [<ffffffff81059558>] update_process_times+0x48/0x90
[74937.224108] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[74937.224108] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[74937.224108] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[74937.224108] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[74937.224108] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[74937.224108] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[74937.224108] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[74937.224108] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[74937.224108] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[74937.224108] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[74937.224108] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[74937.224108] <EOI> [<ffffffff8123260d>] ? radix_tree_lookup_element+0x3d/0x90
[74937.224108] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
[74937.224108] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[74937.224108] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[74937.224108] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[74937.224108] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[74937.224108] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[74937.224108] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[74937.224108] [<ffffffff81159670>] vfs_read+0xb0/0x180
[74937.224108] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[74937.224108] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224105] INFO: rcu_sched self-detected stall on CPU { 0} (t=3975264 jiffies)
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] Call Trace:
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] <EOI> [<ffffffff8123260d>] ? radix_tree_lookup_element+0x3d/0x90
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[75117.236093] INFO: rcu_sched self-detected stall on CPU { 0} (t=4020267 jiffies)
[75117.236093] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
[75117.236093] Call Trace:
[75117.236093] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[75117.236093] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[75117.236093] [<ffffffff81059558>] update_process_times+0x48/0x90
[75117.236093] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[75117.236093] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[75117.236093] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[75117.236093] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[75117.236093] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[75117.236093] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[75117.236093] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[75117.236093] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[75117.236093] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[75117.236093] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[75117.236093] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[75117.236093] <EOI> [<ffffffff81232617>] ? radix_tree_lookup_element+0x47/0x90
[75117.236093] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
[75117.236093] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[75117.236093] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[75117.236093] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[75117.236093] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[75117.236093] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[75117.236093] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[75117.236093] [<ffffffff81159670>] vfs_read+0xb0/0x180
[75117.236093] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[75117.236093] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Our mysql 5.5 database is hosted in a ec2 instance and during one of our daily backup runs, we noticed the following in /var/log/messages, shortly we lost access to the box,so could not dive deeper. Any help will be immensely appreciated!
cac7>] handle_percpu_irq+0x47/0x60
[63596.468086] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[63596.468086] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[63596.468086] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[63596.468086] <EOI> [<ffffffff81232669>] ? radix_tree_lookup_slot+0x9/0x10
[63596.468086] [<ffffffff8123266e>] ? radix_tree_lookup_slot+0xe/0x10
[63596.468086] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[63596.468086] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[63596.468086] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[63596.468086] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[63596.468086] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[63596.468086] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[63596.468086] [<ffffffff81159670>] vfs_read+0xb0/0x180
[63596.468086] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[63596.468086] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] INFO: rcu_sched self-detected stall on CPU { 0} (t=1140075 jiffies)
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] Call Trace:
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff810c984b>] ? handle_irq_event_percpu+0xbb/0x220
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] <EOI> [<ffffffff81232669>] ? radix_tree_lookup_slot+0x9/0x10
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff8123266e>] ? radix_tree_lookup_slot+0xe/0x10
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 03:51:34 ip-10-65-128-4 kernel: [63596.468086] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[68636.804086] INFO: rcu_sched self-detected stall on CPU { 0} (t=2400159 jiffies)
[68636.804086] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62.x86_64 #1
[68636.804086] Call Trace:
[68636.804086] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[68636.804086] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[68636.804086] [<ffffffff81059558>] update_process_times+0x48/0x90
[68636.804086] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[68636.804086] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[68636.804086] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[68636.804086] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[68636.804086] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[68636.804086] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[68636.804086] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[68636.804086] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[68636.804086] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[68636.804086] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[68636.804086] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[68636.804086] <EOI> [<ffffffff812325f8>] ? radix_tree_lookup_element+0x28/0x90
[68636.804086] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
[68636.804086] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[68636.804086] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[68636.804086] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[68636.804086] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[68636.804086] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[68636.804086] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[68636.804086] [<ffffffff81159670>] vfs_read+0xb0/0x180
[68636.804086] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[68636.804086] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] INFO: rcu_sched self-detected stall on CPU { 0} (t=2400159 jiffies)
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] Call Trace:
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] <EOI> [<ffffffff812325f8>] ? radix_tree_lookup_element+0x28/0x90
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 05:15:35 ip-10-65-128-4 kernel: [68636.804086] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[74397.188049] INFO: rcu_sched self-detected stall on CPU { 0} (t=3840255 jiffies)
[74397.188049] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
[74397.188049] Call Trace:
[74397.188049] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[74397.188049] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[74397.188049] [<ffffffff81059558>] update_process_times+0x48/0x90
[74397.188049] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[74397.188049] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[74397.188049] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[74397.188049] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[74397.188049] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[74397.188049] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[74397.188049] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[74397.188049] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[74397.188049] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[74397.188049] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[74397.188049] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[74397.188049] <EOI> [<ffffffff812325ff>] ? radix_tree_lookup_element+0x2f/0x90
[74397.188049] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
[74397.188049] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[74397.188049] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[74397.188049] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[74397.188049] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[74397.188049] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[74397.188049] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[74397.188049] [<ffffffff81159670>] vfs_read+0xb0/0x180
[74397.188049] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[74397.188049] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] INFO: rcu_sched self-detected stall on CPU { 0} (t=3840255 jiffies)
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] Call Trace:
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] <EOI> [<ffffffff812325ff>] ? radix_tree_lookup_element+0x2f/0x90
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 06:51:35 ip-10-65-128-4 kernel: [74397.188049] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[74577.200087] INFO: rcu_sched self-detected stall on CPU { 0} (t=3885258 jiffies)
[74577.200096] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
[74577.200096] Call Trace:
[74577.200096] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[74577.200096] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[74577.200096] [<ffffffff81059558>] update_process_times+0x48/0x90
[74577.200096] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[74577.200096] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[74577.200096] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[74577.200096] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[74577.200096] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[74577.200096] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[74577.200096] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[74577.200096] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[74577.200096] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[74577.200096] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[74577.200096] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[74577.200096] <EOI> [<ffffffff812325d5>] ? radix_tree_lookup_element+0x5/0x90
[74577.200096] [<ffffffff8123266e>] ? radix_tree_lookup_slot+0xe/0x10
[74577.200096] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[74577.200096] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[74577.200096] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[74577.200096] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[74577.200096] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[74577.200096] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[74577.200096] [<ffffffff81159670>] vfs_read+0xb0/0x180
[74577.200096] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[74577.200096] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200087] INFO: rcu_sched self-detected stall on CPU { 0} (t=3885258 jiffies)
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] Call Trace:
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] <EOI> [<ffffffff812325d5>] ? radix_tree_lookup_element+0x5/0x90
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff8123266e>] ? radix_tree_lookup_slot+0xe/0x10
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 06:54:35 ip-10-65-128-4 kernel: [74577.200096] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[74757.212089] INFO: rcu_sched self-detected stall on CPU { 0} (t=3930261 jiffies)
[74757.212089] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
[74757.212089] Call Trace:
[74757.212089] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[74757.212089] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[74757.212089] [<ffffffff81059558>] update_process_times+0x48/0x90
[74757.212089] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[74757.212089] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[74757.212089] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[74757.212089] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[74757.212089] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[74757.212089] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[74757.212089] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[74757.212089] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[74757.212089] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[74757.212089] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[74757.212089] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[74757.212089] <EOI> [<ffffffff812325f8>] ? radix_tree_lookup_element+0x28/0x90
[74757.212089] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
[74757.212089] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[74757.212089] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[74757.212089] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[74757.212089] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[74757.212089] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[74757.212089] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[74757.212089] [<ffffffff81159670>] vfs_read+0xb0/0x180
[74757.212089] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[74757.212089] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] INFO: rcu_sched self-detected stall on CPU { 0} (t=3930261 jiffies)
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] Call Trace:
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] <EOI> [<ffffffff812325f8>] ? radix_tree_lookup_element+0x28/0x90
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 06:57:35 ip-10-65-128-4 kernel: [74757.212089] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[74937.224105] INFO: rcu_sched self-detected stall on CPU { 0} (t=3975264 jiffies)
[74937.224108] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
[74937.224108] Call Trace:
[74937.224108] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[74937.224108] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[74937.224108] [<ffffffff81059558>] update_process_times+0x48/0x90
[74937.224108] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[74937.224108] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[74937.224108] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[74937.224108] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[74937.224108] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[74937.224108] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[74937.224108] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[74937.224108] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[74937.224108] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[74937.224108] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[74937.224108] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[74937.224108] <EOI> [<ffffffff8123260d>] ? radix_tree_lookup_element+0x3d/0x90
[74937.224108] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
[74937.224108] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[74937.224108] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[74937.224108] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[74937.224108] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[74937.224108] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[74937.224108] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[74937.224108] [<ffffffff81159670>] vfs_read+0xb0/0x180
[74937.224108] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[74937.224108] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224105] INFO: rcu_sched self-detected stall on CPU { 0} (t=3975264 jiffies)
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] Call Trace:
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81059558>] update_process_times+0x48/0x90
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] <EOI> [<ffffffff8123260d>] ? radix_tree_lookup_element+0x3d/0x90
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff811037fe>] find_get_page+0x1e/0x90
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81159670>] vfs_read+0xb0/0x180
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
Jul 14 07:00:35 ip-10-65-128-4 kernel: [74937.224108] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b
[75117.236093] INFO: rcu_sched self-detected stall on CPU { 0} (t=4020267 jiffies)
[75117.236093] Pid: 29212, comm: mysqld Tainted: G B O 3.4.62-53.42.x86_64 #1
[75117.236093] Call Trace:
[75117.236093] <IRQ> [<ffffffff810d02aa>] __rcu_pending+0x19a/0x4e0
[75117.236093] [<ffffffff810d0b18>] rcu_check_callbacks+0xb8/0x190
[75117.236093] [<ffffffff81059558>] update_process_times+0x48/0x90
[75117.236093] [<ffffffff81095774>] tick_sched_timer+0x64/0xc0
[75117.236093] [<ffffffff8106f328>] __run_hrtimer+0x88/0x1e0
[75117.236093] [<ffffffff81095710>] ? tick_nohz_handler+0xf0/0xf0
[75117.236093] [<ffffffff8106fc27>] hrtimer_interrupt+0xf7/0x240
[75117.236093] [<ffffffff8100a54b>] xen_timer_interrupt+0x2b/0x140
[75117.236093] [<ffffffff810c97ee>] handle_irq_event_percpu+0x5e/0x220
[75117.236093] [<ffffffff810cbf1e>] ? irq_get_irq_data+0xe/0x10
[75117.236093] [<ffffffff810ccac7>] handle_percpu_irq+0x47/0x60
[75117.236093] [<ffffffff812900dd>] __xen_evtchn_do_upcall+0x2fd/0x310
[75117.236093] [<ffffffff8129193f>] xen_evtchn_do_upcall+0x2f/0x50
[75117.236093] [<ffffffff813f041a>] xen_hvm_callback_vector+0x6a/0x70
[75117.236093] <EOI> [<ffffffff81232617>] ? radix_tree_lookup_element+0x47/0x90
[75117.236093] [<ffffffff8123266e>] radix_tree_lookup_slot+0xe/0x10
[75117.236093] [<ffffffff811037fe>] find_get_page+0x1e/0x90
[75117.236093] [<ffffffff81104ceb>] generic_file_aio_read+0x26b/0x760
[75117.236093] [<ffffffff81104a39>] ? generic_file_aio_write+0x99/0xe0
[75117.236093] [<ffffffff81158d0a>] do_sync_read+0xda/0x120
[75117.236093] [<ffffffff811e21b3>] ? security_file_permission+0x93/0xb0
[75117.236093] [<ffffffff81159191>] ? rw_verify_area+0x61/0xf0
[75117.236093] [<ffffffff81159670>] vfs_read+0xb0/0x180
[75117.236093] [<ffffffff81159902>] sys_pread64+0xa2/0xb0
[75117.236093] [<ffffffff813eeeb9>] system_call_fastpath+0x16/0x1b