lxc-3.0.0 - lxc-copy not working on moblab |
|||
Issue descriptionsudo lxc-copy --lxcpath /mnt/moblab/containers --newpath /mnt/moblab/containers --name moblab_base_07 --newname test_186_1526370154_13036 -s lxc-copy: moblab_base_07: storage/overlay.c: ovl_mount: 650 No such device - Failed to mount "/mnt/moblab/containers/moblab_base_07/rootfs" on "/etc/lxc/rootfs" with options "upperdir=/mnt/moblab/containers/test_186_1526370154_13036/delta0,lowerdir=/mnt/moblab/containers/moblab_base_07/rootfs": No such device
,
May 16 2018
,
May 16 2018
moblab@localhost ~ $ strace sudo lxc-copy --lxcpath /mnt/moblab/containers --newpath /mnt/moblab/containers --name moblab_base_07 --newname test_186_1526370154_13036 -s > out
execve("/usr/bin/sudo", ["sudo", "lxc-copy", "--lxcpath", "/mnt/moblab/containers", "--newpath", "/mnt/moblab/containers", "--name", "moblab_base_07", "--newname", "test_186_1526370154_13036", "-s"],
[/* 22 vars */]) = 0
brk(NULL) = 0x5c0c1935a000
fcntl(0, F_GETFD) = 0
fcntl(1, F_GETFD) = 0
fcntl(2, F_GETFD) = 0
access("/etc/suid-debug", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x767610633000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/usr/libexec/sudo/tls/x86_64/libutil.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/libexec/sudo/tls/x86_64", 0x7ffee55f1690) = -1 ENOENT (No such file or directory)
open("/usr/libexec/sudo/tls/libutil.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/libexec/sudo/tls", 0x7ffee55f1690) = -1 ENOENT (No such file or directory)
open("/usr/libexec/sudo/x86_64/libutil.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/libexec/sudo/x86_64", 0x7ffee55f1690) = -1 ENOENT (No such file or directory)
open("/usr/libexec/sudo/libutil.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/libexec/sudo", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=55244, ...}) = 0
mmap(NULL, 55244, PROT_READ, MAP_PRIVATE, 3, 0) = 0x767610625000
close(3) = 0
open("/lib64/libutil.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=10200, ...}) = 0
mmap(NULL, 2105368, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76761020d000
mprotect(0x76761020f000, 2093056, PROT_NONE) = 0
mmap(0x76761040e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x76761040e000
close(3) = 0
open("/usr/libexec/sudo/libsudo_util.so.0", O_RDONLY|O_CLOEXEC) = 3 [19/1829]
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=88456, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x767610624000
mmap(NULL, 90632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76761060d000
mmap(0x767610622000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x767610622000
close(3) = 0
open("/usr/libexec/sudo/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\10\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1734456, ...}) = 0
mmap(NULL, 3842776, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76760fe62000
mprotect(0x767610003000, 2093056, PROT_NONE) = 0
mmap(0x767610202000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a0000) = 0x767610202000
mmap(0x767610208000, 17112, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x767610208000
close(3) = 0
open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=14448, ...}) = 0
mmap(NULL, 2109584, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76760fc5e000
mprotect(0x76760fc60000, 2097152, PROT_NONE) = 0
mmap(0x76760fe60000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x76760fe60000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x76761060c000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x76761060b000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x76761060a000
arch_prctl(ARCH_SET_FS, 0x76761060b700) = 0
mprotect(0x767610202000, 16384, PROT_READ) = 0
mprotect(0x76760fe60000, 4096, PROT_READ) = 0
mprotect(0x767610622000, 4096, PROT_READ) = 0
mprotect(0x76761040e000, 4096, PROT_READ) = 0
mprotect(0x5c0c18978000, 4096, PROT_READ) = 0
mprotect(0x767610634000, 4096, PROT_READ) = 0
munmap(0x767610625000, 55244) = 0
fcntl(0, F_GETFL) = 0x8002 (flags O_RDWR|O_LARGEFILE)
fcntl(1, F_GETFL) = 0x8001 (flags O_WRONLY|O_LARGEFILE)
fcntl(2, F_GETFL) = 0x8002 (flags O_RDWR|O_LARGEFILE)
brk(NULL) = 0x5c0c1935a000
brk(0x5c0c1937b000) = 0x5c0c1937b000
open("/etc/localtime", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2845, ...}) = 0
fstat(3, {st_mode=S_IFREG|0644, st_size=2845, ...}) = 0
read(3, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\5\0\0\0\5\0\0\0\0"..., 4096) = 2845
lseek(3, -1811, SEEK_CUR) = 1034
read(3, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\5\0\0\0\5\0\0\0\0"..., 4096) = 1811
close(3) = 0
stat("/etc/sudo.conf", 0x7ffee55f0c20) = -1 ENOENT (No such file or directory)
geteuid() = 246
access("/usr/local/bin/sudo", X_OK) = -1 ENOENT (No such file or directory)
access("/usr/bin/sudo", X_OK) = 0
stat("/usr/bin/sudo", {st_mode=S_IFREG|S_ISUID|0711, st_size=112360, ...}) = 0
write(2, "sudo", 4sudo) = 4
write(2, ": ", 2: ) = 2
write(2, "effective uid is not 0, is /usr/"..., 133effective uid is not 0, is /usr/bin/sudo on a file system with the 'nosuid' option set or an NFS file system without root privileges?) = 133
write(2, "\n", 1
) = 1
exit_group(1) = ?
+++ exited with 1 +++
,
May 16 2018
Found some discussion here: https://forum.proxmox.com/threads/pct-start-failed-after-apparmor-update.38479/
,
May 16 2018
This seems pretty definitive:
stat("/usr/bin/sudo", {st_mode=S_IFREG|S_ISUID|0711, st_size=112360, ...}) = 0
write(2, "sudo", 4sudo) = 4
write(2, ": ", 2: ) = 2
write(2, "effective uid is not 0, is /usr/"..., 133effective uid is not 0, is /usr/bin/sudo on a file system with the 'nosuid' option set or an NFS file system without root privileges?) = 133
The program `sudo` is on a filesystem that is mounted with the nosuid option. Can you show the output of the `mount` command on your device?
,
May 16 2018
I am not sure if the sudo messages are a red-herring, sudo worked in lxc-2.1.1 nothing else has changed. I seem to have it working on the fizz based moblab - so it seems like a kernel difference, I am trying to get the strace from the guado and compare to the fizz will update shortly
,
May 16 2018
Here is the strace that does not work (strace_guado) and the strace that does (strace_fizz) biggest different should be kernel version guado is running 3.14 kernel and fizz 4.4 I did a sudo su before running the command so hopefully the sudo does not interfere with the output Guado mount command is : mount /dev/sda3 on / type ext2 (ro,relatime) devtmpfs on /dev type devtmpfs (rw,nosuid,noexec,relatime,size=1986260k,nr_inodes=496565,mode=755) none on /proc type proc (rw,nosuid,nodev,noexec,relatime) none on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) tmp on /tmp type tmpfs (rw,nodev,relatime) run on /run type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755) debugfs on /sys/kernel/debug type debugfs (rw,nosuid,nodev,noexec,relatime,gid=605,mode=750) shmfs on /dev/shm type tmpfs (rw,nosuid,nodev,noexec,relatime) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) /dev/sda1 on /mnt/stateful_partition type ext4 (rw,nosuid,nodev,noexec,noatime,commit=30,data=ordered) /dev/sda8 on /usr/share/oem type ext4 (ro,nosuid,nodev,noexec,relatime,data=ordered) /dev/sda1 on /home type ext4 (rw,nodev,noatime,commit=30,data=ordered) /dev/sda1 on /var type ext4 (rw,nosuid,nodev,noexec,noatime,commit=30,data=ordered) /dev/sda1 on /home/chronos type ext4 (rw,nosuid,nodev,noexec,noatime,commit=30,data=ordered) run on /var/run type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755) run on /var/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755) media on /media type tmpfs (rw,nosuid,nodev,noexec,relatime) /dev/sda1 on /usr/local type ext4 (rw,noatime,commit=30,data=ordered) /dev/sda1 on /var/db/pkg type ext4 (rw,nosuid,nodev,noexec,noatime,commit=30,data=ordered) /dev/sda1 on /var/lib/portage type ext4 (rw,nosuid,nodev,noexec,noatime,commit=30,data=ordered) none on /sys/fs/cgroup type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) pstore on /dev/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/cpu type cgroup (rw,nosuid,nodev,noexec,relatime,cpu) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) imageloader on /run/imageloader type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755) debugfs on /run/debugfs_gpu type debugfs (rw,nosuid,nodev,noexec,relatime,gid=605,mode=750) /dev/sda1 on /mnt/moblab type ext4 (rw,noatime,commit=30,data=ordered) Fizz mount command is : moblab@localhost ~ $ mount /dev/nvme0n1p3 on / type ext2 (ro,relatime,block_validity,barrier,user_xattr,acl) devtmpfs on /dev type devtmpfs (rw,nosuid,noexec,relatime,size=16396508k,nr_inodes=4099127,mode=755) none on /proc type proc (rw,nosuid,nodev,noexec,relatime) none on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) tmp on /tmp type tmpfs (rw,nodev,relatime) run on /run type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755) debugfs on /sys/kernel/debug type debugfs (rw,nosuid,nodev,noexec,relatime,gid=605,mode=750) shmfs on /dev/shm type tmpfs (rw,nosuid,nodev,noexec,relatime) devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) configfs on /sys/kernel/config type configfs (rw,nosuid,nodev,noexec,relatime) /dev/nvme0n1p1 on /mnt/stateful_partition type ext4 (rw,nosuid,nodev,noexec,noatime,commit=30,data=ordered) /dev/nvme0n1p8 on /usr/share/oem type ext4 (ro,nosuid,nodev,noexec,relatime,data=ordered) /dev/nvme0n1p1 on /home type ext4 (rw,nodev,noatime,commit=30,data=ordered) /dev/nvme0n1p1 on /var type ext4 (rw,nosuid,nodev,noexec,noatime,commit=30,data=ordered) /dev/nvme0n1p1 on /home/chronos type ext4 (rw,nosuid,nodev,noexec,noatime,commit=30,data=ordered) run on /var/run type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755) run on /var/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755) media on /media type tmpfs (rw,nosuid,nodev,noexec,relatime) /dev/nvme0n1p1 on /usr/local type ext4 (rw,noatime,commit=30,data=ordered) /dev/nvme0n1p1 on /var/db/pkg type ext4 (rw,nosuid,nodev,noexec,noatime,commit=30,data=ordered) /dev/nvme0n1p1 on /var/lib/portage type ext4 (rw,nosuid,nodev,noexec,noatime,commit=30,data=ordered) tracefs on /sys/kernel/debug/tracing type tracefs (rw,nosuid,nodev,noexec,relatime) pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) none on /sys/fs/cgroup type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755) cgroup on /sys/fs/cgroup/cpu type cgroup (rw,nosuid,nodev,noexec,relatime,cpu) pstore on /dev/pstore type pstore (rw,nosuid,nodev,noexec,relatime) cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices) cgroup on /sys/fs/cgroup/cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct) cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) imageloader on /run/imageloader type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755) debugfs on /run/debugfs_gpu type debugfs (rw,nosuid,nodev,noexec,relatime,gid=605,mode=750) /dev/nvme0n1p1 on /mnt/moblab type ext4 (rw,noatime,commit=30,data=ordered)
,
May 16 2018
This looks like the error: lxc-copy: moblab_base_07: storage/overlay.c: ovl_mount: 650 No such device - Failed to mount "/mnt/moblab/containers/moblab_base_07/rootfs" on "/etc/lxc/rootfs" with options "upperdir=/mnt/moblab/containers/test_186_1526370154_13036/delta0,lowerdir=/mnt/moblab/containers/moblab_base_07/rootfs": No such device The thing that looks suspicious to me is /etc/lxc/rootfs. Does that directory exist? Otherwise, can you re-run the strace with the -f option so that it will also trace child processes?
,
May 16 2018
strace -f attached Also since the mount fails: localhost /mnt/moblab/containers # ls /mnt/moblab/containers/moblab_base_07/rootfs bin boot dev etc home lib lib64 media mnt opt proc root run sbin srv sys tmp usr var localhost /mnt/moblab/containers # ls -lrt total 294788 drwxr-xr-- 3 root root 4096 May 16 14:02 moblab_base_06 -rw-r--r-- 1 moblab moblab 301852988 May 16 14:03 moblab_base_07.tar.xz drwxrwx--- 3 root root 4096 May 16 14:57 moblab_base_07 localhost /mnt/moblab/containers # ls -lrt /mnt/moblab/containers/moblab_base_07/rootfs total 76 drwxr-xr-x 2 root root 4096 Apr 12 2016 sys drwxr-xr-x 2 root root 4096 Apr 12 2016 proc drwxr-xr-x 2 root root 4096 Apr 12 2016 boot drwxr-xr-x 10 root root 4096 May 14 00:43 usr drwxr-xr-x 2 root root 4096 May 14 00:43 srv drwxr-xr-x 2 root root 4096 May 14 00:43 opt drwxr-xr-x 2 root root 4096 May 14 00:43 mnt drwxr-xr-x 2 root root 4096 May 14 00:43 media drwxr-xr-x 4 root root 4096 May 14 00:44 dev drwxr-xr-x 12 root root 4096 May 14 00:44 var drwxr-xr-x 2 root root 4096 May 14 00:44 lib64 drwxr-xr-x 11 root root 4096 May 14 00:45 lib drwxr-xr-x 2 root root 4096 May 14 00:45 sbin drwxr-xr-x 2 root root 4096 May 14 00:45 bin drwxr-xr-x 3 root root 4096 May 14 00:45 home drwxr-xr-x 2 root root 4096 May 14 00:45 run drwxr-xr-x 79 root root 4096 May 16 12:19 etc drwxrwxrwt 7 root root 4096 May 16 14:05 tmp drwx------ 2 root root 4096 May 16 14:05 root localhost /mnt/moblab/containers # ls -lrt /etc/lxc/rootfs total 4 -rw-r--r-- 1 root root 189 May 15 18:12 README localhost /mnt/moblab/containers # ls -lrt /mnt/moblab/containers/ total 294788 drwxr-xr-- 3 root root 4096 May 16 14:02 moblab_base_06 -rw-r--r-- 1 moblab moblab 301852988 May 16 14:03 moblab_base_07.tar.xz drwxrwx--- 3 root root 4096 May 16 14:57 moblab_base_07
,
May 16 2018
This is the error in the child process:
[pid 8377] mount("/mnt/moblab/containers/moblab_base_07/rootfs", "/etc/lxc/rootfs", "overlay", MS_MGC_VAL, "upperdir=/mnt/moblab/containers/"...) = -1 ENODEV (No such device)
[pid 8377] mount("/mnt/moblab/containers/moblab_base_07/rootfs", "/etc/lxc/rootfs", "overlayfs", MS_MGC_VAL, "upperdir=/mnt/moblab/containers/"...) = -1 ENODEV (No such device)
[pid 8377] mount("/mnt/moblab/containers/moblab_base_07/rootfs", "/etc/lxc/rootfs", "overlay", MS_MGC_VAL, "upperdir=/mnt/moblab/containers/"...) = -1 ENODEV (No such device)
[pid 8377] mount("/mnt/moblab/containers/moblab_base_07/rootfs", "/etc/lxc/rootfs", "overlayfs", MS_MGC_VAL, "upperdir=/mnt/moblab/containers/"...) = -1 ENODEV (No such device)
According to the manpage for mount (http://man7.org/linux/man-pages/man2/mount.2.html), ENODEV means the filesystem type is not available in the kernel.
Is the guado kernel built with overlayfs support? overlayfs first landed in upstream in version 3.18 so unless someone has backported it to 3.14 this is not going to work.
,
May 17 2018
I am going to close out this bug- this was not the best logging in the world and me getting confused with the fact the fizz has overlayfs and the guado does not ( the lxc code actually handles this, it falls back to non snapshot copy if the snapshot fails ) The problem with lxc 3 not working on moblab is this line of code. https://cs.corp.google.com/chromeos_public/src/third_party/autotest/files/site_utils/lxc/container.py?rcl=0b22f6e2d46d0530ff38a4332614ed28d449c9cc&l=288 lxc.rootfs needs to be changed to lxc.rootfs.path I have an email out the the maintainer of that code to ask how he wants the change to be implemented, since it needs to be backward compatible, when that is done I think moblab will be good with the lxc 3
,
May 17 2018
Follow up bug at https://bugs.chromium.org/p/chromium/issues/detail?id=844050 |
|||
►
Sign in to add a comment |
|||
Comment 1 by haddowk@chromium.org
, May 16 2018