On the Linux kernel mailing list (LKML) a reproducible crash of a kernel built with clang has been reported: https://lkml.org/lkml/2018/4/24/1227 

I think what happens is this:

- func1 stores a pointer a function storing a pointer in the caller-saved register RCX
- func1 calls func2, without saving RCX on the stack
- func2 clobbers RCX and returns
- func1 tries to dereference the pointer in RCX, which now doesn't point to a valid address

To reproduce:

cd ${KERNEL_REPO}
git fetch https://chromium.googlesource.com/chromiumos/third_party/kernel/ refs/sandbox/mka/llvm/v4.14 && git checkout FETCH_HEAD

cp ${ATTACHED_CONFIG} .config
make CC=clang oldconfig
make CC=clang -j 48 bzImage

KPATH=${KERNEL_REPO}/arch/x86/boot
sudo qemu-system-x86_64 -enable-kvm -M pc -kernel $KPATH/bzImage -m 512 -net none -serial stdio -append "root=/dev/ram0 console=ttyS0 hung_task_panic=1 earlyprintk=ttyS0,115200"

This results in a crash like this:

[    0.004000] general protection fault: 0000 [#1] SMP
[    0.004000] Modules linked in:
[    0.004000] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 4.14.0+ #10
[    0.004000] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1 04/01/2014
[    0.004000] task: ffffffffbb410480 task.stack: ffffffffbb400000
[    0.004000] RIP: 0010:irq_work_tick+0xaf/0x120
[    0.004000] RSP: 0000:ffff96fb5fc03e58 EFLAGS: 00010046
[    0.004000] RAX: 0000000000000082 RBX: ffff96fb5fc11d80 RCX: 47e992c1bc778d00
[    0.004000] RDX: 000000000001ba00 RSI: fffffffffffffed4 RDI: ffff96fb5fc1ba38
[    0.004000] RBP: ffff96fb5fc03e78 R08: 0000000000000000 R09: 0000000000000018
[    0.004000] R10: 0000000000000000 R11: 0000000000000018 R12: ffffffffbb41bd40
[    0.004000] R13: 0000000000000000 R14: ffffffffbb410480 R15: 0000000000000000
[    0.004000] FS:  0000000000000000(0000) GS:ffff96fb5fc00000(0000) knlGS:0000000000000000
[    0.004000] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[    0.004000] CR2: 00000000ffffffff CR3: 000000001ec09000 CR4: 00000000000006b0
[    0.004000] Call Trace:
[    0.004000]  <IRQ>
[    0.004000]  update_process_times+0x6e/0xa0
[    0.004000]  tick_periodic+0x78/0x90
[    0.004000]  tick_handle_periodic+0x26/0x80
[    0.004000]  timer_interrupt+0x13/0x20
[    0.004000]  __handle_irq_event_percpu+0x106/0x230
[    0.004000]  handle_irq_event+0x5a/0xc0
[    0.004000]  handle_level_irq+0x11a/0x190
[    0.004000]  handle_irq+0x1f/0x30
[    0.004000]  do_IRQ+0x4b/0xd0
[    0.004000]  common_interrupt+0x93/0x93
[    0.004000]  </IRQ>
[    0.004000] RIP: 0010:native_restore_fl+0xd/0x20
[    0.004000] RSP: 0000:ffffffffbb403e08 EFLAGS: 00000282 ORIG_RAX: ffffffffffffffcf
[    0.004000] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000000
[    0.004000] RDX: ffff96fb5f01b800 RSI: 0000000000000282 RDI: 0000000000000282
[    0.004000] RBP: ffffffffbb403e10 R08: 0000000000000001 R09: 000000000000003f
[    0.004000] R10: ffff96fb5f01b800 R11: 0000000000000007 R12: ffff96fb5f01ba08
[    0.004000] R13: ffff96fb5f01b8e0 R14: ffffffffbb41bd40 R15: ffff96fb5f01b800
[    0.004000]  _raw_spin_unlock_irqrestore+0x1a/0x20
[    0.004000]  __setup_irq+0x610/0x7b0
[    0.004000]  setup_irq+0x5b/0x90
[    0.004000]  hpet_time_init+0x32/0x40
[    0.004000]  x86_late_time_init+0x10/0x20
[    0.004000]  start_kernel+0x460/0x580
[    0.004000]  x86_64_start_kernel+0x30f/0x320
[    0.004000]  secondary_startup_64+0xa5/0xa5
[    0.004000] Code: 4c 0f b1 7b f8 4c 89 e3 4d 85 e4 75 cf 48 c7 c1 88 5b 01 00 65 48 03 0c 25 d8 a1 00 00 ff 14 25 10 08 42 bb a9 00 02 00 00 75 4b <48> 83 39 00 74 3c 31 db 48 87 19 48 85 db 74 32 90 48 8d 7b f8
[    0.004000] RIP: irq_work_tick+0xaf/0x120 RSP: ffff96fb5fc03e58
[    0.004000] ---[ end trace 08945838e05bf5b2 ]---
[    0.004000] Kernel panic - not syncing: Fatal exception in interrupt
[    0.004000] ---[ end Kernel panic - not syncing: Fatal exception in interrupt

The exception occurs at 'irq_work_tick+0xaf'

objdump -d -S --start-address=0x$(grep irq_work_tick System.map | sed -e
 "s/ \+.*//") vmlinux | less

...
void irq_work_tick(void)
{
ffffffff81193200:       55                      push   %rbp
...
static inline struct llist_node *llist_del_all(struct llist_head *head)
{
        return xchg(&head->first, NULL);
ffffffff8119324a:       48 87 19                xchg   %rbx,(%rcx)
        while (llnode != NULL) {
ffffffff8119324d:       48 85 db                test   %rbx,%rbx
ffffffff81193250:       74 3f                   je     ffffffff81193291 <irq_work_tick+0x91>
ffffffff81193252:       0f 1f 40 00             nopl   0x0(%rax)
ffffffff81193256:       66 2e 0f 1f 84 00 00    nopw   %cs:0x0(%rax,%rax,1)
ffffffff8119325d:       00 00 00
                work = llist_entry(llnode, struct irq_work, llnode);
ffffffff81193260:       48 8d 7b f8             lea    -0x8(%rbx),%rdi
                flags = work->flags & ~IRQ_WORK_PENDING;
ffffffff81193264:       4c 8b 7b f8             mov    -0x8(%rbx),%r15
        return node->next;
ffffffff81193268:       4c 8b 23                mov    (%rbx),%r12
ffffffff8119326b:       4d 89 fe                mov    %r15,%r14
ffffffff8119326e:       49 83 e6 fe             and    $0xfffffffffffffffe,%r14
                xchg(&work->flags, flags);
ffffffff81193272:       4c 89 f0                mov    %r14,%rax
ffffffff81193275:       48 87 43 f8             xchg   %rax,-0x8(%rbx)
                work->func(work);
ffffffff81193279:       ff 53 08                callq  *0x8(%rbx)
                (void)cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY);
ffffffff8119327c:       49 83 e7 fc             and    $0xfffffffffffffffc,%r15
ffffffff81193280:       4c 89 f0                mov    %r14,%rax
ffffffff81193283:       f0 4c 0f b1 7b f8       lock cmpxchg %r15,-0x8(%rbx)
ffffffff81193289:       4c 89 e3                mov    %r12,%rbx
        while (llnode != NULL) {
ffffffff8119328c:       4d 85 e4                test   %r12,%r12
ffffffff8119328f:       75 cf                   jne    ffffffff81193260 <irq_work_tick+0x60>
                irq_work_run_list(raised);
        irq_work_run_list(this_cpu_ptr(&lazy_list));
ffffffff81193291:       48 c7 c1 88 5b 01 00    mov    $0x15b88,%rcx
ffffffff81193298:       65 48 03 0c 25 d8 a1    add    %gs:0xa1d8,%rcx
ffffffff8119329f:       00 00
ffffffff811932a1:       ff 14 25 10 08 c2 81    callq  *0xffffffff81c20810
        BUG_ON(!irqs_disabled());
ffffffff811932a8:       a9 00 02 00 00          test   $0x200,%eax
ffffffff811932ad:       75 4b                   jne    ffffffff811932fa <irq_work_tick+0xfa>
        return ACCESS_ONCE(head->first) == NULL;
ffffffff811932af:       48 83 39 00             cmpq   $0x0,(%rcx)


My interpretation is:

'irq_work_tick+0xaf' corresponds to address 0xffffffff811932af, so the
exception occurs during the execution of 'cmpq   $0x0,(%rcx)' or
'ACCESS_ONCE(head->first) == NULL'. The instruction checks if the
memory location in RCX contains 0. Supposedly the register should
contain the address of 'head->first', however the value is
0x47e992c1bc778d00, which doesn't look like a valid address.

RCX is set shortly before:

ffffffff81193291:       48 c7 c1 88 5b 01 00    mov    $0x15b88,%rcx
ffffffff81193298:       65 48 03 0c 25 d8 a1    add    %gs:0xa1d8,%rcx

As stated in https://www.kernel.org/doc/Documentation/this_cpu_ops.txt
the GS segment register is used by the kernel for per-cpu
variables. And effectively System.map tells us the offset 0x15b88
corresponds to the per-cpu variable lazy_list:

0000000000015b88 d lazy_list

The register dump shows GS with a value of 0xffff96fb5fc00000, which
looks reasonable.

But wait, right after setting RCX we do this:

ffffffff811932a1:       ff 14 25 10 08 c2 81    callq  *0xffffffff81c20810
        BUG_ON(!irqs_disabled());

According to my (limited) understanding of x86 calling conventions RCX
is a caller-saved register, thus the caller should save it on the
stack to preserve its value across a function call.
(https://en.wikipedia.org/wiki/X86_calling_conventions)

The call to '*0xffffffff81c20810' appears to be related with
KVM/paravirtualization:

grep ffffffff81c20810 System.map
ffffffff81c20810 D pv_irq_ops

Since there is no offset it must be calling the first function in the
structure, which is 'safe_fl' and points to 'native_save_fl'
(https://elixir.bootlin.com/linux/v4.14.36/source/arch/x86/kernel/paravirt.c#L316)

objdump -d -S --start-address=0x$(grep native_save_fl System.map | sed
 -e "s/ \+.*//") vmlinux | less

static inline unsigned long native_save_fl(void)
{
ffffffff81060240:       55                      push   %rbp
ffffffff81060241:       48 89 e5                mov    %rsp,%rbp
ffffffff81060244:       48 83 ec 10             sub    $0x10,%rsp
ffffffff81060248:       65 48 8b 04 25 28 00    mov    %gs:0x28,%rax
ffffffff8106024f:       00 00
ffffffff81060251:       48 89 45 f8             mov    %rax,-0x8(%rbp)
        /*
         * "=rm" is safe here, because "pop" adjusts the stack before
         * it evaluates its effective address -- this is part of the
         * documented behavior of the "pop" instruction.
         */
        asm volatile("# __raw_save_flags\n\t"
ffffffff81060255:       9c                      pushfq
ffffffff81060256:       8f 45 f0                popq   -0x10(%rbp)
                     "pushf ; pop %0"
                     : "=rm" (flags)
                     : /* no input */
                     : "memory");

        return flags;
ffffffff81060259:       48 8b 45 f0             mov    -0x10(%rbp),%rax
ffffffff8106025d:       65 48 8b 0c 25 28 00    mov    %gs:0x28,%rcx
ffffffff81060264:       00 00
ffffffff81060266:       48 3b 4d f8             cmp    -0x8(%rbp),%rcx
ffffffff8106026a:       75 06                   jne    ffffffff81060272 <native_save_fl+0x32>
ffffffff8106026c:       48 83 c4 10             add    $0x10,%rsp
ffffffff81060270:       5d                      pop    %rbp
ffffffff81060271:       c3                      retq
ffffffff81060272:       e8 09 ec 01 00          callq  ffffffff8107ee80 <__stack_chk_fail>
ffffffff81060277:       66 0f 1f 84 00 00 00    nopw   0x0(%rax,%rax,1)
ffffffff8106027e:       00 0

At 0xffffffff8106025d this clobbers RCX.

(I'm no expert in x86/assembly, please correct me if my analysis is wrong)

Currently this isn't known to affect Chrome OS, but I'm concerned that it might hit us later.
 

Deleted: .config 191 KB

.config 191 KB Download