butler and/or kitchen is crashing on windows |
|||||
Issue descriptionhttps://chromium-swarm-dev.appspot.com/task?id=34eef5d3c63b1410&refresh=10&show_raw=1 Crashytimes: runtime: nelems=170 nfree=152 nalloc=18 previous allocCount=17 nfreed=65535 fatal error: sweep increased allocation count runtime stack: runtime.throw(0xa90856, 0x20) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/panic.go:596 +0x9c runtime.(*mspan).sweep(0x2a86580, 0x2a86500, 0xc042020680) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/mgcsweep.go:288 +0x794 runtime.sweepone(0x432aed) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/mgcsweep.go:110 +0x1bb runtime.gosweepone.func1() E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/mgcsweep.go:125 +0x32 runtime.systemstack(0xc042018c00) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/asm_amd64.s:327 +0x7e runtime.mstart() E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/proc.go:1132 goroutine 3 [running]: runtime.systemstack_switch() E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/asm_amd64.s:281 fp=0xc042029f68 sp=0xc042029f60 runtime.gosweepone(0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/mgcsweep.go:126 +0x5a fp=0xc042029f98 sp=0xc042029f68 runtime.bgsweep(0xc042034000) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/mgcsweep.go:59 +0xc9 fp=0xc042029fd8 sp=0xc042029f98 runtime.goexit() E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc042029fe0 sp=0xc042029fd8 created by runtime.gcenable E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/mgc.go:212 +0x68 goroutine 1 [semacquire, 2 minutes]: sync.runtime_Semacquire(0xc04223e1d4) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/sema.go:47 +0x3b sync.(*WaitGroup).Wait(0xc04223e1c8) E:/b/rr/tmpfvvgry/w/golang/go/src/sync/waitgroup.go:131 +0x81 github.com/luci/luci-go/logdog/client/butler.(*Butler).Wait(0xc04223e180, 0xc04229f350, 0x20) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:288 +0x1e5 main.(*cookRun).runWithLogdogButler.func2(0xc04223e180, 0x2a882c8, 0xc042126f00, 0xc04229f9b0) E:/b/rr/tmpfvvgry/w/infra/go/src/infra/tools/kitchen/cook_logdog.go:301 +0x44 main.(*cookRun).runWithLogdogButler(0xc042064540, 0x2a882c8, 0xc042126f00, 0xc042064638, 0xc042138a20, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/src/infra/tools/kitchen/cook_logdog.go:419 +0x19cc main.(*cookRun).ensureAndRun(0xc042064540, 0x2a882c8, 0xc042126f00, 0xc042138a20, 0xc0421252d0, 0x1, 0x1) E:/b/rr/tmpfvvgry/w/infra/go/src/infra/tools/kitchen/cook.go:254 +0x89c main.(*cookRun).runErr(0xc042064540, 0x2a882c8, 0xc042126f00, 0xc04204e270, 0x0, 0x0, 0xc042138a20, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/src/infra/tools/kitchen/cook.go:439 +0x2f5 main.(*cookRun).Run(0xc042064540, 0xd6b420, 0xd99760, 0xc04204e270, 0x0, 0x0, 0xc0421385d0, 0xc042124fe8) E:/b/rr/tmpfvvgry/w/infra/go/src/infra/tools/kitchen/cook.go:368 +0x12f github.com/maruel/subcommands.Run(0xd6b420, 0xd99760, 0xc04204e150, 0x13, 0x13, 0xc042020000) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/maruel/subcommands/subcommands.go:360 +0x4ea main.main() E:/b/rr/tmpfvvgry/w/infra/go/src/infra/tools/kitchen/kitchen.go:50 +0x8c goroutine 5 [syscall, 6 minutes]: os/signal.signal_recv(0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/sigqueue.go:116 +0x10b os/signal.loop() E:/b/rr/tmpfvvgry/w/golang/go/src/os/signal/signal_unix.go:22 +0x29 created by os/signal.init.1 E:/b/rr/tmpfvvgry/w/golang/go/src/os/signal/signal_unix.go:28 +0x48 goroutine 6 [chan receive, 6 minutes]: main.handleInterruption.func1(0xc04200ad20, 0xc042125060) E:/b/rr/tmpfvvgry/w/infra/go/src/infra/tools/kitchen/kitchen.go:59 +0x64 created by main.handleInterruption E:/b/rr/tmpfvvgry/w/infra/go/src/infra/tools/kitchen/kitchen.go:66 +0xfc goroutine 9 [semacquire, 2 minutes]: sync.runtime_Semacquire(0xc04217dc84) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/sema.go:47 +0x3b sync.(*WaitGroup).Wait(0xc04217dc78) E:/b/rr/tmpfvvgry/w/golang/go/src/sync/waitgroup.go:131 +0x81 github.com/Microsoft/go-winio.(*win32File).closeHandle(0xc04217dc70) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/file.go:96 +0x66 github.com/Microsoft/go-winio.(*win32File).Close(0xc04217dc70, 0xc0422642a0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/file.go:105 +0x32 github.com/Microsoft/go-winio.(*win32PipeListener).listenerRoutine(0xc0421956c0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/pipe.go:277 +0x3ed created by github.com/Microsoft/go-winio.ListenPipe E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/pipe.go:352 +0x2ec goroutine 18 [chan send (nil chan), 2 minutes]: github.com/Microsoft/go-winio.ioCompletionProcessor(0xdc) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/file.go:133 +0xea created by github.com/Microsoft/go-winio.initIo E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/file.go:57 +0x87 goroutine 4680 [runnable]: net.(*netFD).connect.func2(0xd68bc0, 0xc04200a7e0, 0xc0423d28c0, 0xc04200a840) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:376 +0x164 created by net.(*netFD).connect E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:384 +0x1f2 goroutine 10 [chan receive, 6 minutes]: github.com/Microsoft/go-winio.(*win32PipeListener).Accept(0xc0421956c0, 0xc042195650, 0xa8e8b5, 0x1e, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/pipe.go:373 +0x120 github.com/luci/luci-go/logdog/client/butler/streamserver.(*listenerStreamServer).serve(0xc042195650) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/streamserver/base.go:128 +0xf5 created by github.com/luci/luci-go/logdog/client/butler/streamserver.(*listenerStreamServer).Listen E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/streamserver/base.go:88 +0x1be goroutine 19 [chan receive, 6 minutes]: github.com/Microsoft/go-winio.(*win32File).asyncIo(0xc04217dc70, 0xc0421fa120, 0x0, 0xc000000000, 0x0, 0x0, 0xd62040, 0xc0421c0860, 0x0, 0x0, ...) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/file.go:167 +0x24d github.com/Microsoft/go-winio.connectPipe(0xc04217dc70, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/pipe.go:362 +0xc1 github.com/Microsoft/go-winio.(*win32PipeListener).listenerRoutine.func1(0xc0421eea80, 0xc0421c2018) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/pipe.go:267 +0x35 created by github.com/Microsoft/go-winio.(*win32PipeListener).listenerRoutine E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/pipe.go:268 +0x286 goroutine 130 [IO wait, 3 minutes]: net.runtime_pollWait(0x2b20b40, 0x72, 0xd5c550) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/netpoll.go:164 +0x60 net.(*pollDesc).wait(0xc0423d61a0, 0x72, 0xd5c550, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_poll_runtime.go:75 +0x3f net.(*ioSrv).ExecIO(0xc0421c2020, 0xc0423d6060, 0xa7d8e8, 0x7, 0xaa3e68, 0x11, 0x3f3, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:196 +0xfe net.(*netFD).Read(0xc0423d6000, 0xc042037400, 0x400, 0x400, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:452 +0x13d net.(*conn).Read(0xc042004330, 0xc042037400, 0x400, 0x400, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/net.go:181 +0x77 crypto/tls.(*block).readFromUntil(0xc042252ae0, 0x2b21170, 0xc042004330, 0x5, 0xc042004330, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:488 +0x9f crypto/tls.(*Conn).readRecord(0xc042170e00, 0xaa4617, 0xc042170f20, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:590 +0xcb crypto/tls.(*Conn).Read(0xc042170e00, 0xc042532000, 0x8000, 0x8000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:1134 +0x124 bufio.(*Reader).Read(0xc04250c660, 0xc042454738, 0x9, 0x9, 0xc042413d50, 0x404462, 0x42d15b) E:/b/rr/tmpfvvgry/w/golang/go/src/bufio/bufio.go:213 +0x319 io.ReadAtLeast(0xd5ddc0, 0xc04250c660, 0xc042454738, 0x9, 0x9, 0x9, 0x404179, 0xc04250c6c0, 0xc042217d10) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:307 +0xb0 io.ReadFull(0xd5ddc0, 0xc04250c660, 0xc042454738, 0x9, 0x9, 0xc042016600, 0xc042173380, 0xc0421c6080) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:325 +0x5f golang.org/x/net/http2.readFrameHeader(0xc042454738, 0x9, 0x9, 0xd5ddc0, 0xc04250c660, 0x0, 0xc000000000, 0xc04250c6c0, 0xc042173380) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:237 +0x82 golang.org/x/net/http2.(*Framer).ReadFrame(0xc042454700, 0xc0423bd950, 0xd5f500, 0xc042aaa7c5, 0xc042aaa7c5) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:492 +0xab google.golang.org/grpc/transport.(*framer).readFrame(0xc0423bd860, 0xc0428dfa00, 0xc0428dfa00, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http_util.go:508 +0x36 google.golang.org/grpc/transport.(*http2Client).reader(0xc0423fa180) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1039 +0xd4 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:251 +0xcc1 goroutine 162 [select, 2 minutes]: google.golang.org/grpc/transport.(*http2Client).controller(0xc0423f8300) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1112 +0x661 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:281 +0x1088 goroutine 35 [chan receive, 6 minutes]: google.golang.org/api/internal.(*PoolResolver).Next(0xc042222c00, 0xc042242ba0, 0xc0422e3f68, 0xc042039b00, 0x8, 0x8) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/api/internal/pool.go:54 +0x6b google.golang.org/grpc.(*roundRobin).watchAddrUpdates(0xc0422429c0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/balancer.go:168 +0x6b google.golang.org/grpc.(*roundRobin).Start.func1(0xc0422429c0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/balancer.go:239 +0x32 created by google.golang.org/grpc.(*roundRobin).Start E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/balancer.go:243 +0x15a goroutine 44 [chan receive, 6 minutes]: google.golang.org/grpc.(*ClientConn).lbWatcher(0xc04237a300) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:473 +0xac created by google.golang.org/grpc.DialContext E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:416 +0x543 goroutine 37 [select, 6 minutes]: google.golang.org/grpc.(*addrConn).transportMonitor(0xc04203a680) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:861 +0x66d google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc04203a680) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:606 +0x1e0 created by google.golang.org/grpc.(*ClientConn).resetAddrConn E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:607 +0x379 goroutine 38 [select, 6 minutes]: google.golang.org/grpc.(*addrConn).transportMonitor(0xc04203a9c0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:861 +0x66d google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc04203a9c0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:606 +0x1e0 created by google.golang.org/grpc.(*ClientConn).resetAddrConn E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:607 +0x379 goroutine 39 [select, 6 minutes]: google.golang.org/grpc.(*addrConn).transportMonitor(0xc04203ad00) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:861 +0x66d google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc04203ad00) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:606 +0x1e0 created by google.golang.org/grpc.(*ClientConn).resetAddrConn E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:607 +0x379 goroutine 40 [select, 6 minutes]: google.golang.org/grpc.(*addrConn).transportMonitor(0xc04203b040) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:861 +0x66d google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc04203b040) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:606 +0x1e0 created by google.golang.org/grpc.(*ClientConn).resetAddrConn E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:607 +0x379 goroutine 41 [select, 6 minutes]: google.golang.org/grpc.(*addrConn).transportMonitor(0xc04203b380) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:861 +0x66d google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc04203b380) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:606 +0x1e0 created by google.golang.org/grpc.(*ClientConn).resetAddrConn E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:607 +0x379 goroutine 42 [select, 6 minutes]: google.golang.org/grpc.(*addrConn).transportMonitor(0xc04203b6c0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:861 +0x66d google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc04203b6c0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:606 +0x1e0 created by google.golang.org/grpc.(*ClientConn).resetAddrConn E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:607 +0x379 goroutine 114 [IO wait, 3 minutes]: net.runtime_pollWait(0x2b20a80, 0x72, 0xd5c550) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/netpoll.go:164 +0x60 net.(*pollDesc).wait(0xc0423d81a0, 0x72, 0xd5c550, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_poll_runtime.go:75 +0x3f net.(*ioSrv).ExecIO(0xc0421c2020, 0xc0423d8060, 0xa7d8e8, 0x7, 0xaa3e68, 0x11, 0xff3, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:196 +0xfe net.(*netFD).Read(0xc0423d8000, 0xc042436000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:452 +0x13d net.(*conn).Read(0xc042280028, 0xc042436000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/net.go:181 +0x77 crypto/tls.(*block).readFromUntil(0xc04240a120, 0x2b21170, 0xc042280028, 0x5, 0xc042280028, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:488 +0x9f crypto/tls.(*Conn).readRecord(0xc0423fe000, 0xaa4617, 0xc0423fe120, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:590 +0xcb crypto/tls.(*Conn).Read(0xc0423fe000, 0xc042328000, 0x8000, 0x8000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:1134 +0x124 bufio.(*Reader).Read(0xc04200a6c0, 0xc0421221f8, 0x9, 0x9, 0xc04220dd50, 0x404462, 0x42d15b) E:/b/rr/tmpfvvgry/w/golang/go/src/bufio/bufio.go:213 +0x319 io.ReadAtLeast(0xd5ddc0, 0xc04200a6c0, 0xc0421221f8, 0x9, 0x9, 0x9, 0x404179, 0xc04200a780, 0xc04200c280) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:307 +0xb0 io.ReadFull(0xd5ddc0, 0xc04200a6c0, 0xc0421221f8, 0x9, 0x9, 0xc04201a000, 0xc0422904e0, 0xc04224a440) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:325 +0x5f golang.org/x/net/http2.readFrameHeader(0xc0421221f8, 0x9, 0x9, 0xd5ddc0, 0xc04200a6c0, 0x0, 0xc000000000, 0xc04200a780, 0xc0422904e0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:237 +0x82 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0421221c0, 0xc042050840, 0xd5f500, 0xc0422b40b0, 0xc0422b40b0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:492 +0xab google.golang.org/grpc/transport.(*framer).readFrame(0xc042050480, 0xc042500220, 0xc042500220, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http_util.go:508 +0x36 google.golang.org/grpc/transport.(*http2Client).reader(0xc042416180) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1039 +0xd4 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:251 +0xcc1 goroutine 96 [chan receive, 2 minutes]: github.com/luci/luci-go/logdog/client/butler.New.func1.1(0xc04228cd80) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:215 +0x61 github.com/luci/luci-go/common/sync/parallel.runImpl.func1(0xc04228cd80, 0xc0422140e0) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:229 +0x5d created by github.com/luci/luci-go/common/sync/parallel.runImpl E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:230 +0x95 goroutine 95 [chan receive, 3 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody(0xc0422b7680, 0x10, 0xc0423abfa8, 0xc042125f50) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:119 +0xaa github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoop(0xc0422b7680, 0x10, 0x10) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:104 +0x104 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).init.func1 E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:92 +0xcf goroutine 93 [IO wait, 3 minutes]: net.runtime_pollWait(0x2b20c00, 0x72, 0xd5c550) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/netpoll.go:164 +0x60 net.(*pollDesc).wait(0xc0423d41a0, 0x72, 0xd5c550, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_poll_runtime.go:75 +0x3f net.(*ioSrv).ExecIO(0xc0421c2020, 0xc0423d4060, 0xa7d8e8, 0x7, 0xaa3e68, 0x11, 0xff3, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:196 +0xfe net.(*netFD).Read(0xc0423d4000, 0xc042492000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:452 +0x13d net.(*conn).Read(0xc04240c000, 0xc042492000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/net.go:181 +0x77 crypto/tls.(*block).readFromUntil(0xc0423bc1e0, 0x2b21170, 0xc04240c000, 0x5, 0xc04240c000, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:488 +0x9f crypto/tls.(*Conn).readRecord(0xc042418000, 0xaa4617, 0xc042418120, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:590 +0xcb crypto/tls.(*Conn).Read(0xc042418000, 0xc042524000, 0x8000, 0x8000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:1134 +0x124 bufio.(*Reader).Read(0xc04228cba0, 0xc04250e818, 0x9, 0x9, 0xc04247fd50, 0x404462, 0x42d15b) E:/b/rr/tmpfvvgry/w/golang/go/src/bufio/bufio.go:213 +0x319 io.ReadAtLeast(0xd5ddc0, 0xc04228cba0, 0xc04250e818, 0x9, 0x9, 0x9, 0x404179, 0xc04228cc00, 0xc04234e1e0) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:307 +0xb0 io.ReadFull(0xd5ddc0, 0xc04228cba0, 0xc04250e818, 0x9, 0x9, 0xc042017900, 0xc0423b4340, 0xc0422881c0) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:325 +0x5f golang.org/x/net/http2.readFrameHeader(0xc04250e818, 0x9, 0x9, 0xd5ddc0, 0xc04228cba0, 0x0, 0xc000000000, 0xc04228cc00, 0xc0423b4340) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:237 +0x82 golang.org/x/net/http2.(*Framer).ReadFrame(0xc04250e7e0, 0xc0422b75f0, 0xd5f500, 0xc042362840, 0xc042362840) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:492 +0xab google.golang.org/grpc/transport.(*framer).readFrame(0xc0422b7500, 0xc04251d100, 0xc04251d100, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http_util.go:508 +0x36 google.golang.org/grpc/transport.(*http2Client).reader(0xc04237a480) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1039 +0xd4 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:251 +0xcc1 goroutine 115 [select, 3 minutes]: google.golang.org/grpc/transport.(*http2Client).controller(0xc042416180) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1112 +0x661 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:281 +0x1088 goroutine 91 [IO wait, 3 minutes]: net.runtime_pollWait(0x2b209c0, 0x72, 0xd5c550) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/netpoll.go:164 +0x60 net.(*pollDesc).wait(0xc04235e520, 0x72, 0xd5c550, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_poll_runtime.go:75 +0x3f net.(*ioSrv).ExecIO(0xc0421c2020, 0xc04235e3e0, 0xa7d8e8, 0x7, 0xaa3e68, 0x11, 0x3f3, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:196 +0xfe net.(*netFD).Read(0xc04235e380, 0xc042408400, 0x400, 0x400, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:452 +0x13d net.(*conn).Read(0xc0423f4000, 0xc042408400, 0x400, 0x400, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/net.go:181 +0x77 crypto/tls.(*block).readFromUntil(0xc0423bc1b0, 0x2b21170, 0xc0423f4000, 0x5, 0xc0423f4000, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:488 +0x9f crypto/tls.(*Conn).readRecord(0xc0423fc000, 0xaa4617, 0xc0423fc120, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:590 +0xcb crypto/tls.(*Conn).Read(0xc0423fc000, 0xc0424d8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:1134 +0x124 bufio.(*Reader).Read(0xc04228c900, 0xc04250e658, 0x9, 0x9, 0xc0423e1d50, 0x404462, 0x42d15b) E:/b/rr/tmpfvvgry/w/golang/go/src/bufio/bufio.go:213 +0x319 io.ReadAtLeast(0xd5ddc0, 0xc04228c900, 0xc04250e658, 0x9, 0x9, 0x9, 0x404179, 0xc04228c960, 0xc0422ccdc0) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:307 +0xb0 io.ReadFull(0xd5ddc0, 0xc04228c900, 0xc04250e658, 0x9, 0x9, 0xc042015300, 0xc0423da1a0, 0xc042350740) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:325 +0x5f golang.org/x/net/http2.readFrameHeader(0xc04250e658, 0x9, 0x9, 0xd5ddc0, 0xc04228c900, 0x0, 0xc000000000, 0xc04228c960, 0xc0423da1a0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:237 +0x82 golang.org/x/net/http2.(*Framer).ReadFrame(0xc04250e620, 0xc0422b7410, 0xd5f500, 0xc04245f735, 0xc04245f735) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:492 +0xab google.golang.org/grpc/transport.(*framer).readFrame(0xc0422b7320, 0xc042ac7520, 0xc042ac7520, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http_util.go:508 +0x36 google.golang.org/grpc/transport.(*http2Client).reader(0xc04237a000) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1039 +0xd4 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:251 +0xcc1 goroutine 163 [IO wait, 3 minutes]: net.runtime_pollWait(0x2b20840, 0x72, 0xd5c550) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/netpoll.go:164 +0x60 net.(*pollDesc).wait(0xc0423d2360, 0x72, 0xd5c550, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_poll_runtime.go:75 +0x3f net.(*ioSrv).ExecIO(0xc0421c2020, 0xc0423d2220, 0xa7d8e8, 0x7, 0xaa3e68, 0x11, 0x3f3, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:196 +0xfe net.(*netFD).Read(0xc0423d21c0, 0xc0423b7000, 0x400, 0x400, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:452 +0x13d net.(*conn).Read(0xc0423e8008, 0xc0423b7000, 0x400, 0x400, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/net.go:181 +0x77 crypto/tls.(*block).readFromUntil(0xc0422b63c0, 0x2b21170, 0xc0423e8008, 0x5, 0xc0423e8008, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:488 +0x9f crypto/tls.(*Conn).readRecord(0xc0423ee380, 0xaa4617, 0xc0423ee4a0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:590 +0xcb crypto/tls.(*Conn).Read(0xc0423ee380, 0xc0424b6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:1134 +0x124 bufio.(*Reader).Read(0xc04241aae0, 0xc04221c738, 0x9, 0x9, 0xc0424cbd50, 0x404462, 0x42d15b) E:/b/rr/tmpfvvgry/w/golang/go/src/bufio/bufio.go:213 +0x319 io.ReadAtLeast(0xd5ddc0, 0xc04241aae0, 0xc04221c738, 0x9, 0x9, 0x9, 0x404179, 0xc04241ab40, 0xc0422cce60) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:307 +0xb0 io.ReadFull(0xd5ddc0, 0xc04241aae0, 0xc04221c738, 0x9, 0x9, 0xc042017900, 0xc0422a4680, 0xc0423507c0) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:325 +0x5f golang.org/x/net/http2.readFrameHeader(0xc04221c738, 0x9, 0x9, 0xd5ddc0, 0xc04241aae0, 0x0, 0xc000000000, 0xc04241ab40, 0xc0422a4680) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:237 +0x82 golang.org/x/net/http2.(*Framer).ReadFrame(0xc04221c700, 0xc0424ed350, 0xd5f500, 0xc0422b4bf0, 0xc0422b4bf0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:492 +0xab google.golang.org/grpc/transport.(*framer).readFrame(0xc0424ed260, 0xc042555560, 0xc042555560, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http_util.go:508 +0x36 google.golang.org/grpc/transport.(*http2Client).reader(0xc0423f8480) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1039 +0xd4 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:251 +0xcc1 goroutine 161 [IO wait, 2 minutes]: net.runtime_pollWait(0x2b20900, 0x72, 0xd5c550) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/netpoll.go:164 +0x60 net.(*pollDesc).wait(0xc042064a60, 0x72, 0xd5c550, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_poll_runtime.go:75 +0x3f net.(*ioSrv).ExecIO(0xc0421c2020, 0xc042064920, 0xa7d8e8, 0x7, 0xaa3e68, 0x11, 0x3f3, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:196 +0xfe net.(*netFD).Read(0xc0420648c0, 0xc0422a3000, 0x400, 0x400, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:452 +0x13d net.(*conn).Read(0xc042280030, 0xc0422a3000, 0x400, 0x400, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/net.go:181 +0x77 crypto/tls.(*block).readFromUntil(0xc04227ca20, 0x2b21170, 0xc042280030, 0x5, 0xc042280030, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:488 +0x9f crypto/tls.(*Conn).readRecord(0xc0423fe380, 0xaa4617, 0xc0423fe4a0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:590 +0xcb crypto/tls.(*Conn).Read(0xc0423fe380, 0xc0424be000, 0x8000, 0x8000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:1134 +0x124 bufio.(*Reader).Read(0xc04241a7e0, 0xc04221c578, 0x9, 0x9, 0xc042279d50, 0x404462, 0x42d15b) E:/b/rr/tmpfvvgry/w/golang/go/src/bufio/bufio.go:213 +0x319 io.ReadAtLeast(0xd5ddc0, 0xc04241a7e0, 0xc04221c578, 0x9, 0x9, 0x9, 0x404179, 0xc04241a840, 0xc0421a5680) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:307 +0xb0 io.ReadFull(0xd5ddc0, 0xc04241a7e0, 0xc04221c578, 0x9, 0x9, 0xc042014000, 0xc042290820, 0xc0421f2d00) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:325 +0x5f golang.org/x/net/http2.readFrameHeader(0xc04221c578, 0x9, 0x9, 0xd5ddc0, 0xc04241a7e0, 0x0, 0xc000000000, 0xc04241a840, 0xc042290820) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:237 +0x82 golang.org/x/net/http2.(*Framer).ReadFrame(0xc04221c540, 0xc0424ed110, 0xd5f500, 0xc042556a50, 0xc042556a50) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:492 +0xab google.golang.org/grpc/transport.(*framer).readFrame(0xc0424ed020, 0xc042283860, 0xc042283860, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http_util.go:508 +0x36 google.golang.org/grpc/transport.(*http2Client).reader(0xc0423f8300) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1039 +0xd4 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:251 +0xcc1 goroutine 164 [select, 3 minutes]: google.golang.org/grpc/transport.(*http2Client).controller(0xc0423f8480) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1112 +0x661 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:281 +0x1088 goroutine 131 [select, 3 minutes]: google.golang.org/grpc/transport.(*http2Client).controller(0xc0423fa180) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1112 +0x661 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:281 +0x1088 goroutine 92 [select, 3 minutes]: google.golang.org/grpc/transport.(*http2Client).controller(0xc04237a000) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1112 +0x661 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:281 +0x1088 goroutine 94 [select, 3 minutes]: google.golang.org/grpc/transport.(*http2Client).controller(0xc04237a480) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1112 +0x661 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:281 +0x1088 goroutine 132 [semacquire, 2 minutes]: sync.runtime_notifyListWait(0xc042127850, 0xc000000898) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/sema.go:297 +0x119 sync.(*Cond).Wait(0xc042127840) E:/b/rr/tmpfvvgry/w/golang/go/src/sync/cond.go:57 +0x90 github.com/luci/luci-go/common/sync/cancelcond.(*Cond).Wait(0xc042244030, 0x2a84070, 0xc0420102f8, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/cancelcond/cancelCond.go:69 +0x1ff github.com/luci/luci-go/logdog/client/butler/bundler.(*Bundler).makeBundles(0xc042127800) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/bundler/bundler.go:298 +0x54f created by github.com/luci/luci-go/logdog/client/butler/bundler.New E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/bundler/bundler.go:92 +0x21a goroutine 133 [chan receive, 2 minutes]: github.com/luci/luci-go/common/sync/parallel.Ignore(0xc04228cd20) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/consumers.go:22 +0x5a github.com/luci/luci-go/logdog/client/butler.New.func1(0xc04223e180, 0xc0423ca1b0) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:225 +0xb8 created by github.com/luci/luci-go/logdog/client/butler.New E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:226 +0x48e goroutine 4617 [select, 2 minutes]: github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams(0xc04223e180, 0xc042a9c240) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:498 +0x372 github.com/luci/luci-go/logdog/client/butler.(*Butler).Wait.func1(0xc042a9c1e0, 0xc04223e180, 0xc042a9c240) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:284 +0x62 created by github.com/luci/luci-go/logdog/client/butler.(*Butler).Wait E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:285 +0x1b9 goroutine 135 [select, 6 minutes]: github.com/luci/luci-go/logdog/client/butler.New.func3(0xc04223e180) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:236 +0x2ae created by github.com/luci/luci-go/logdog/client/butler.New E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:243 +0x4d2 goroutine 136 [chan receive, 6 minutes]: github.com/luci/luci-go/logdog/client/butler/streamserver.(*listenerStreamServer).Next(0xc042195650, 0xaa36e0, 0xc042483f90, 0x1) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/streamserver/base.go:93 +0x59 github.com/luci/luci-go/logdog/client/butler.(*Butler).AddStreamServer.func1(0xc04250cc60, 0xc04223e180, 0xd69040, 0xc042195650, 0x2a88288, 0xc0423bde60) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:359 +0xd5 created by github.com/luci/luci-go/logdog/client/butler.(*Butler).AddStreamServer E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:380 +0x13d goroutine 137 [chan receive, 2 minutes]: github.com/Microsoft/go-winio.(*win32PipeListener).Close(0xc0421956c0, 0xa93d0a, 0x26) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/github.com/Microsoft/go-winio/pipe.go:392 +0xe8 github.com/luci/luci-go/logdog/client/butler/streamserver.(*listenerStreamServer).Close(0xc042195650) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/streamserver/base.go:109 +0x54 github.com/luci/luci-go/logdog/client/butler.(*Butler).AddStreamServer.func2(0xc04223e180, 0x2a88288, 0xc0423bde60, 0xd69040, 0xc042195650, 0xc04250cc60) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:390 +0xdb created by github.com/luci/luci-go/logdog/client/butler.(*Butler).AddStreamServer E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:392 +0x1ae goroutine 177 [chan receive, 2 minutes]: github.com/luci/luci-go/common/sync/parallel.runImpl.func2(0xc0422140f0, 0xc04228cd20, 0xc04228cd80, 0xc04228cc60) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:251 +0xf8 created by github.com/luci/luci-go/common/sync/parallel.runImpl E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:259 +0xd5 goroutine 76 [chan receive, 3 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0xc042125f50, 0xc04230d1c0, 0x121601, 0xc0422b7680) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0xbe created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0x158 goroutine 169 [chan receive, 2 minutes]: cloud.google.com/go/pubsub.(*Topic).initBundler.func1(0xc042300360, 0xc042242f00, 0x2a84070, 0xc0420103c0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/cloud.google.com/go/pubsub/topic.go:253 +0x67 created by cloud.google.com/go/pubsub.(*Topic).initBundler E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/cloud.google.com/go/pubsub/topic.go:256 +0xea goroutine 170 [select, 2 minutes]: google.golang.org/api/support/bundler.(*Bundler).background(0xc0421cc000) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/api/support/bundler/bundler.go:270 +0x488 created by google.golang.org/api/support/bundler.NewBundler E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/api/support/bundler/bundler.go:117 +0x2d1 goroutine 183 [chan receive, 3 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0xc042125f50, 0xc0423649e0, 0x121601, 0xc0422b7680) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0xbe created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0x158 goroutine 591 [chan receive, 3 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0xc042125f50, 0xc04254a1e0, 0x121601, 0xc0422b7680) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0xbe created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0x158 goroutine 4592 [select, 2 minutes]: github.com/luci/luci-go/common/sync/cancelcond.(*Cond).Wait.func1(0xc042242540, 0x2a84070, 0xc0420102f8, 0xc042556960, 0xc042244030, 0xc0422424e0) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/cancelcond/cancelCond.go:54 +0x1a3 created by github.com/luci/luci-go/common/sync/cancelcond.(*Cond).Wait E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/cancelcond/cancelCond.go:63 +0x1be goroutine 3080 [chan receive, 2 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0xc042125f50, 0xc04254a640, 0x121601, 0xc0422b7680) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0xbe created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0x158 goroutine 2819 [chan receive, 4 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0xc042125f50, 0xc042aecac0, 0x121601, 0xc0422b7680) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0xbe created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0x158 goroutine 3186 [chan receive, 4 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0xc042125f50, 0xc04254a840, 0x121601, 0xc0422b7680) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0xbe created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0x158 goroutine 1942 [chan receive, 3 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0xc042125f50, 0xc042406e80, 0x121601, 0xc0422b7680) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0xbe created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0x158 goroutine 4047 [chan receive, 3 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0xc042125f50, 0xc0422820a0, 0x121601, 0xc0422b7680) E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0xbe created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpfvvgry/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0x158 goroutine 4463 [runnable, locked to thread]: syscall.Syscall6(0x7fefe30dd30, 0x5, 0x4d4, 0x6, 0x1, 0xc04255d294, 0x4, 0x0, 0xc04255d2c0, 0x40f3bc, ...) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/syscall_windows.go:174 +0x6b syscall.Setsockopt(0x4d4, 0x100000006, 0xc04255d294, 0x4, 0xc04255d2b0, 0xc04255d2a8) E:/b/rr/tmpfvvgry/w/golang/go/src/syscall/zsyscall_windows.go:1420 +0xa9 syscall.SetsockoptInt(0x4d4, 0x6, 0x1, 0x1, 0x40fc2f, 0x8) E:/b/rr/tmpfvvgry/w/golang/go/src/syscall/syscall_windows.go:710 +0x5f net.setNoDelay(0xc0423d28c0, 0xc042280001, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/tcpsockopt_posix.go:19 +0xb6 net.newTCPConn(0xc0423d28c0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/tcpsock.go:180 +0x61 net.doDialTCP(0xd68bc0, 0xc04200a7e0, 0xa7ac0c, 0x3, 0x0, 0xc0428b0060, 0xdbf508, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/tcpsock_posix.go:98 +0x2e6 net.dialTCP(0xd68bc0, 0xc04200a7e0, 0xa7ac0c, 0x3, 0x0, 0xc0428b0060, 0x0, 0x0, 0xc04255d4c0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/tcpsock_posix.go:58 +0xeb net.dialSingle(0xd68bc0, 0xc04200a7e0, 0xc04205e700, 0xd63980, 0xc0428b0060, 0x0, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/dial.go:521 +0x480 net.dialSerial(0xd68bc0, 0xc04250c2a0, 0xc04205e700, 0xc04205e300, 0x6, 0x8, 0x0, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/dial.go:489 +0x227 net.(*Dialer).DialContext(0xc04255d7a8, 0xd68bc0, 0xc04250c2a0, 0xa7ac0c, 0x3, 0xa8b3fc, 0x19, 0x0, 0x0, 0x0, ...) E:/b/rr/tmpfvvgry/w/golang/go/src/net/dial.go:371 +0x712 google.golang.org/grpc/transport.dialContext(0x2b25568, 0xc04250c2a0, 0xa7ac0c, 0x3, 0xa8b3fc, 0x19, 0x0, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/go17.go:45 +0xde google.golang.org/grpc/transport.dial(0x2b25568, 0xc04250c2a0, 0x0, 0xa8b3fc, 0x19, 0x4, 0x3, 0xc042a75900, 0x40ee7d) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:130 +0xe3 google.golang.org/grpc/transport.newHTTP2Client(0x2b25568, 0xc04250c2a0, 0xa8b3fc, 0x19, 0x981180, 0xc042214910, 0x0, 0x0, 0x0, 0x0, ...) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:166 +0xc6 google.golang.org/grpc/transport.NewClientTransport(0x2b25568, 0xc04250c2a0, 0xa8b3fc, 0x19, 0x981180, 0xc042214910, 0x0, 0x0, 0x0, 0x0, ...) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/transport.go:407 +0xa8 google.golang.org/grpc.(*addrConn).resetTransport(0xc0428ba680, 0xd62700, 0xc0422421e0, 0xc0422421f8) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:799 +0x3ba google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc0428ba680) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:598 +0x3b created by google.golang.org/grpc.(*ClientConn).resetAddrConn E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:607 +0x379 goroutine 4673 [select, 2 minutes]: google.golang.org/grpc.(*addrConn).transportMonitor(0xc0428f9a00) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:861 +0x66d google.golang.org/grpc.(*ClientConn).resetAddrConn.func1(0xc0428f9a00) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:606 +0x1e0 created by google.golang.org/grpc.(*ClientConn).resetAddrConn E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/clientconn.go:607 +0x379 goroutine 4679 [select, 2 minutes]: google.golang.org/grpc/transport.(*http2Client).controller(0xc0422de300) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1112 +0x661 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:281 +0x1088 goroutine 4678 [IO wait, 2 minutes]: net.runtime_pollWait(0x2b20cc0, 0x72, 0xd5c550) E:/b/rr/tmpfvvgry/w/golang/go/src/runtime/netpoll.go:164 +0x60 net.(*pollDesc).wait(0xc0423d2520, 0x72, 0xd5c550, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_poll_runtime.go:75 +0x3f net.(*ioSrv).ExecIO(0xc0421c2020, 0xc0423d23e0, 0xa7d8e8, 0x7, 0xaa3e68, 0x9, 0xff3, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:196 +0xfe net.(*netFD).Read(0xc0423d2380, 0xc0428e6000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/fd_windows.go:452 +0x13d net.(*conn).Read(0xc0422440f8, 0xc0428e6000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/net/net.go:181 +0x77 crypto/tls.(*block).readFromUntil(0xc04259b560, 0x2b21170, 0xc0422440f8, 0x5, 0xc0422440f8, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:488 +0x9f crypto/tls.(*Conn).readRecord(0xc0422dc380, 0xaa4617, 0xc0422dc4a0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:590 +0xcb crypto/tls.(*Conn).Read(0xc0422dc380, 0xc042960000, 0x8000, 0x8000, 0x0, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/golang/go/src/crypto/tls/conn.go:1134 +0x124 bufio.(*Reader).Read(0xc042243860, 0xc0421d4658, 0x9, 0x9, 0x9, 0x0, 0x18) E:/b/rr/tmpfvvgry/w/golang/go/src/bufio/bufio.go:213 +0x319 io.ReadAtLeast(0xd5ddc0, 0xc042243860, 0xc0421d4658, 0x9, 0x9, 0x9, 0xc042343db8, 0x40fc2f, 0x30) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:307 +0xb0 io.ReadFull(0xd5ddc0, 0xc042243860, 0xc0421d4658, 0x9, 0x9, 0x1201ec0000000000, 0x1200000000000000, 0x1) E:/b/rr/tmpfvvgry/w/golang/go/src/io/io.go:325 +0x5f golang.org/x/net/http2.readFrameHeader(0xc0421d4658, 0x9, 0x9, 0xd5ddc0, 0xc042243860, 0x0, 0x0, 0xc0428b0690, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:237 +0x82 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0421d4620, 0xd63280, 0xc0428b0690, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/golang.org/x/net/http2/frame.go:492 +0xab google.golang.org/grpc/transport.(*framer).readFrame(0xc0428e2f00, 0xc0428b0690, 0xc0428b0690, 0x0, 0x0) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http_util.go:508 +0x36 google.golang.org/grpc/transport.(*http2Client).reader(0xc0422de300) E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:1039 +0xd4 created by google.golang.org/grpc/transport.newHTTP2Client E:/b/rr/tmpfvvgry/w/infra/go/.vendor/src/google.golang.org/grpc/transport/http2_client.go:251 +0xcc1
,
Mar 16 2017
This is showing up continuously on https://luci-scheduler-dev.appspot.com/jobs/infra/infra-continuous-win-64
,
Mar 16 2017
I see lots of tasks fail with BOT_DIED error. It usually indicates a bug in Swarming itself, not Kitchen.
,
Mar 16 2017
there is no kitchen exit code. There would be if kitchen crashed and swarming catched it, right?
,
Mar 18 2017
I have opened issue 702843 for BOT_DIED error.
,
Mar 18 2017
Closing this bug because bug 702843 was created for the cause of this bug
,
Mar 24 2017
This bug was closed in error. It seems like the first broken build started with the Go 1.8 push: https://chromium.googlesource.com/infra/infra/+/ead16fde8ad7f4c755e73436e273b090aa2c4d0c Either this is triggering a bug in Go 1.8, or Go 1.8 revealed a latent bug in the existing code base. Benefit of the doubt goes to the latter theory. Likely culprit, since this is only crashing on Windows, is the "winio" package. I see two immediate options: 1) Switch back to Go 1.8 w/ "npipe" implementation and see if that fixes things, then figure out what's wrong with "winio". 2) Revert Go 1.8. ATM we're in a state where no upstream changes can be rolled, and that's not acceptable. Thoughts?
,
Mar 24 2017
The following revision refers to this bug: https://chromium.googlesource.com/external/github.com/luci/luci-go.git/+/6cb138dd7f4d900c66b22f22b9bbde1b0c4ff975 commit 6cb138dd7f4d900c66b22f22b9bbde1b0c4ff975 Author: dnj <dnj@chromium.org> Date: Fri Mar 24 02:10:32 2017 Switch from "winio" to "npipe". This is a hopeful fix to the linked bug. The problem is Go runtime panic that manifested in coordination with an upgrade to Go 1.8. There are several GitHub issue threads referencing similar issues with Windows and "winio". We have to disable the test because "npipe" is actually not a fully- correct Go Listener implementation, and the test fails because of this. This is okay, though, as this code has run for quite a while on Windows using "npipe" without issue. BUG= chromium:702105 TEST=None R=vadimsh@chromium.org Review-Url: https://codereview.chromium.org/2770013003 [modify] https://crrev.com/6cb138dd7f4d900c66b22f22b9bbde1b0c4ff975/logdog/client/butler/streamserver/namedPipe_windows.go [modify] https://crrev.com/6cb138dd7f4d900c66b22f22b9bbde1b0c4ff975/logdog/client/butler/streamserver/namedPipe_windows_test.go [modify] https://crrev.com/6cb138dd7f4d900c66b22f22b9bbde1b0c4ff975/logdog/client/butlerlib/streamclient/client_namedPipe_windows.go
,
Mar 24 2017
The following revision refers to this bug: https://chromium.googlesource.com/infra/infra/+/73d24743affa06713b00bbcf33b33743341ccc9e commit 73d24743affa06713b00bbcf33b33743341ccc9e Author: Dan Jacques <dnj@chromium.org> Date: Fri Mar 24 02:47:13 2017 Roll luci-go. Roll luci-go to pick up "npipe" change. infra/go/src/github.com/luci/luci-go: 6cb138dd Switch from "winio" to "npipe". 11dcd139 auth: Protect auth token creation with local mutex. cca3eb7f auth: Add WithLocalMutex method to Cache interface. 7d2962bd auth: Cache auth tokens for outbound requests in the request context. d3713a53 auth: Decouple tokenCache implementation from auth config in the context. f2880da7 milo: fix buildbucket.parseBuilderQuery e82cab1e auth: Use the token server to generate delegation tokens. 12104d6a Milo: Use luci-config for storing buildbot acls f3bc112b Milo: Split pubsub and the prpc api out to its own modules BUG= chromium:702105 TEST=None Change-Id: I42a4f95daffede10e2f2c896720c510b7e1a1964 Reviewed-on: https://chromium-review.googlesource.com/459078 Reviewed-by: Vadim Shtayura <vadimsh@chromium.org> Commit-Queue: Daniel Jacques <dnj@chromium.org> [modify] https://crrev.com/73d24743affa06713b00bbcf33b33743341ccc9e/DEPS
,
Mar 24 2017
Reverting back to "npipe" seems to have cleared up the builds: https://luci-milo-dev.appspot.com/swarming/task/351720a2ed0e6b10?server=chromium-swarm-dev.appspot.com
,
Mar 24 2017
Reverting back to "npipe" seems to have cleared up the builds: https://luci-milo-dev.appspot.com/swarming/task/351720a2ed0e6b10?server=chromium-swarm-dev.appspot.com I'm going to lower this to P2, since it's working and using a library ("npipe") that's known to work. This will be fun to track down if I have a nice block of time in the future...
,
Apr 5 2017
|
|||||
►
Sign in to add a comment |
|||||
Comment 1 by iannu...@google.com
, Mar 16 2017