Logdog OOM on large json output |
||||||||
Issue descriptionRelevant build: https://uberchromegw.corp.google.com/i/chromium.perf/builders/Win%207%20x64%20Perf/builds/37 The build ends up running twice because logdog fails during its execution, and remote_run re-runs. We probably shouldn't do that; it leads to weird situations like this where the build gets run twice in the same build. Stack trace: fatal error: out of memory runtime stack: runtime.throw(0x9e8960, 0xd) E:/b/rr/tmpidromi/w/golang/go/src/runtime/panic.go:547 +0x7f runtime.(*mcache).refill(0x160250, 0xa, 0x140f1d40) E:/b/rr/tmpidromi/w/golang/go/src/runtime/mcache.go:121 +0xce runtime.mallocgc.func2() E:/b/rr/tmpidromi/w/golang/go/src/runtime/malloc.go:642 +0x2b runtime.systemstack(0x11696000) E:/b/rr/tmpidromi/w/golang/go/src/runtime/asm_386.s:313 +0x5e runtime.mstart() E:/b/rr/tmpidromi/w/golang/go/src/runtime/proc.go:1051 goroutine 1154778 [running]: runtime.systemstack_switch() E:/b/rr/tmpidromi/w/golang/go/src/runtime/asm_386.s:267 fp=0x3bf53e38 sp=0x3bf53e34 runtime.mallocgc(0x90, 0x93d740, 0x0, 0x4) E:/b/rr/tmpidromi/w/golang/go/src/runtime/malloc.go:643 +0x6ca fp=0x3bf53ea4 sp=0x3bf53e38 runtime.newarray(0x93d740, 0x1, 0xce8d0d54) E:/b/rr/tmpidromi/w/golang/go/src/runtime/malloc.go:798 +0xbc fp=0x3bf53ec8 sp=0x3bf53ea4 runtime.mapassign1(0x9763e0, 0x6447bfe0, 0x3bf53f98, 0x3bf53f90) E:/b/rr/tmpidromi/w/golang/go/src/runtime/hashmap.go:453 +0xe9 fp=0x3bf53f28 sp=0x3bf53ec8 github.com/luci/luci-go/logdog/client/butler.(*stream).readChunk(0x590f3580, 0xad1600) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/stream.go:40 +0x231 fp=0x3bf53fc0 sp=0x3bf53f28 github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2.2(0x5b7ba1c0, 0x590f3580) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:450 +0x43 fp=0x3bf53fd0 sp=0x3bf53fc0 runtime.goexit() E:/b/rr/tmpidromi/w/golang/go/src/runtime/asm_386.s:1585 +0x1 fp=0x3bf53fd4 sp=0x3bf53fd0 created by github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2 E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:452 +0xad goroutine 1 [chan receive, 195 minutes]: github.com/luci/luci-go/common/system/ctxcmd.(*CtxCmd).Wait(0x11769050, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/system/ctxcmd/proc.go:189 +0x11f main.(*runCommandRun).Run.func2(0x331e4c58, 0x11806ec0, 0x117d5260, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/cmd/logdog_butler/subcommand_run.go:317 +0x5ad main.(*application).runWithButler(0x11794100, 0x331e4c80, 0x11794100, 0x331e6688, 0x117a9b80, 0x11843c04, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/cmd/logdog_butler/main.go:195 +0x3ad main.(*runCommandRun).Run(0x117909a0, 0x331e4d10, 0x11794100, 0x116921f0, 0x12, 0x12, 0x0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/cmd/logdog_butler/subcommand_run.go:341 +0x1a0b github.com/maruel/subcommands.Run(0x331e4d10, 0x11794100, 0x116921a8, 0x1b, 0x1b, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/github.com/maruel/subcommands/subcommands.go:262 +0x32e main.mainImpl(0x331e48a0, 0x117b2c20, 0x11692148, 0x27, 0x27, 0x0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/cmd/logdog_butler/main.go:324 +0xfff main.main.func2() E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/cmd/logdog_butler/main.go:341 +0x73 github.com/luci/luci-go/common/runtime/paniccatcher.Do(0x11843f94, 0x11843f84) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/runtime/paniccatcher/catch.go:45 +0x3f main.main() E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/cmd/logdog_butler/main.go:347 +0xf0 goroutine 5 [syscall, 195 minutes]: os/signal.signal_recv(0x0) E:/b/rr/tmpidromi/w/golang/go/src/runtime/sigqueue.go:116 +0x11e os/signal.loop() E:/b/rr/tmpidromi/w/golang/go/src/os/signal/signal_unix.go:22 +0x1a created by os/signal.init.1 E:/b/rr/tmpidromi/w/golang/go/src/os/signal/signal_unix.go:28 +0x36 goroutine 18 [syscall, locked to thread]: syscall.Syscall(0x75dd1136, 0x2, 0x504, 0xffffffff, 0x0, 0x1169cea4, 0x3e5, 0x290030) E:/b/rr/tmpidromi/w/golang/go/src/runtime/syscall_windows.go:163 +0x4a syscall.WaitForSingleObject(0x504, 0xffffffff, 0x1169ceb8, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/syscall/zsyscall_windows.go:693 +0x5f gopkg.in/natefinch/npipe%2ev2.waitForCompletion(0x508, 0x5b8205e0, 0x117b3a20, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:182 +0x3a gopkg.in/natefinch/npipe%2ev2.(*PipeListener).AcceptPipe(0x117b3a20, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:327 +0x33f gopkg.in/natefinch/npipe%2ev2.(*PipeListener).Accept(0x117b3a20, 0x0, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:263 +0x39 github.com/luci/luci-go/logdog/client/butler/streamserver.(*listenerStreamServer).serve(0x11768db0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/streamserver/base.go:116 +0xea created by github.com/luci/luci-go/logdog/client/butler/streamserver.(*listenerStreamServer).Listen E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/streamserver/base.go:76 +0x1bf goroutine 17 [chan receive, 195 minutes]: main.mainImpl.func2(0x11770c40, 0x11794100) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/cmd/logdog_butler/main.go:297 +0x4c created by main.mainImpl E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/cmd/logdog_butler/main.go:309 +0xc74 goroutine 29 [chan receive]: github.com/luci/luci-go/common/sync/cancelcond.(*Cond).Wait.func2(0x64470480, 0x644704c0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/cancelcond/cancelCond.go:66 +0x3d github.com/luci/luci-go/common/sync/cancelcond.(*Cond).Wait(0x1176dce0, 0x33710140, 0x64478540, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/cancelcond/cancelCond.go:70 +0x18f github.com/luci/luci-go/logdog/client/butler/bundler.(*Bundler).makeBundles(0x11808a50) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/bundler/bundler.go:264 +0xa5d created by github.com/luci/luci-go/logdog/client/butler/bundler.New E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/bundler/bundler.go:90 +0x1de goroutine 30 [chan receive]: github.com/luci/luci-go/common/sync/parallel.Ignore(0x11765680) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/consumers.go:22 +0x43 github.com/luci/luci-go/logdog/client/butler.New.func1(0x117d5260, 0x11810440) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:190 +0x88 created by github.com/luci/luci-go/logdog/client/butler.New E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:191 +0x479 goroutine 42 [chan send, 12 minutes]: github.com/luci/luci-go/common/sync/parallel.(Semaphore).Lock-fm() E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:104 +0x44 github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody(0x11863cc0, 0x10, 0x117e0fbc, 0x1175f760) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:118 +0x29 github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoop(0x11863cc0, 0x10, 0x10) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:104 +0xcb created by github.com/luci/luci-go/common/sync/parallel.(*Runner).init.func1 E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:92 +0xb4 goroutine 43 [chan receive]: github.com/luci/luci-go/logdog/client/butler.New.func1.1(0x117656c0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:180 +0x4a github.com/luci/luci-go/common/sync/parallel.runImpl.func1(0x117656c0, 0x1175f750) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:229 +0x46 created by github.com/luci/luci-go/common/sync/parallel.runImpl E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:230 +0x82 goroutine 31 [select]: github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams(0x117d5260, 0x11810580) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:421 +0x2be github.com/luci/luci-go/logdog/client/butler.New.func2(0x117d5260) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:196 +0x4d created by github.com/luci/luci-go/logdog/client/butler.New E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:197 +0x496 goroutine 32 [select, 195 minutes]: github.com/luci/luci-go/logdog/client/butler.New.func3(0x117d5260) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:201 +0x24e created by github.com/luci/luci-go/logdog/client/butler.New E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:208 +0x4b3 goroutine 49 [chan receive]: github.com/luci/luci-go/logdog/client/butler/streamserver.(*listenerStreamServer).Next(0x11768db0, 0x0, 0x0, 0x11682300) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/streamserver/base.go:81 +0x46 github.com/luci/luci-go/logdog/client/butler.(*Butler).AddStreamServer.func1(0x11810680, 0x117d5260, 0x331e4ea8, 0x11768db0, 0x331e48a0, 0x11806ea0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:307 +0xa9 created by github.com/luci/luci-go/logdog/client/butler.(*Butler).AddStreamServer E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:328 +0x113 goroutine 50 [chan receive, 195 minutes]: github.com/luci/luci-go/logdog/client/butler.(*Butler).AddStreamServer.func2(0x117d5260, 0x331e48a0, 0x11806ea0, 0x331e4ea8, 0x11768db0, 0x11810680) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:336 +0x6e created by github.com/luci/luci-go/logdog/client/butler.(*Butler).AddStreamServer E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:340 +0x176 goroutine 51 [select, 195 minutes]: golang.org/x/net/context.propagateCancel.func1(0x331e4c80, 0x11794100, 0x331e4c38, 0x11806ec0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/golang.org/x/net/context/pre_go17.go:97 +0x117 created by golang.org/x/net/context.propagateCancel E:/b/rr/tmpidromi/w/infra/go/.vendor/src/golang.org/x/net/context/pre_go17.go:102 +0x1b2 goroutine 44 [chan receive]: github.com/luci/luci-go/common/sync/parallel.runImpl.func2(0x1175f758, 0x11765680, 0x117656c0, 0x11765600) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:251 +0xec created by github.com/luci/luci-go/common/sync/parallel.runImpl E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:259 +0xb7 goroutine 52 [syscall, 195 minutes, locked to thread]: syscall.Syscall(0x75dd1136, 0x2, 0x3ac, 0xffffffff, 0x0, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/runtime/syscall_windows.go:163 +0x4a syscall.WaitForSingleObject(0x3ac, 0xffffffff, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/syscall/zsyscall_windows.go:693 +0x5f os.(*Process).wait(0x11803c40, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/os/exec_windows.go:18 +0x8b os.(*Process).Wait(0x11803c40, 0x1179b860, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/os/doc.go:49 +0x2b os/exec.(*Cmd).Wait(0x117aa820, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/os/exec/exec.go:396 +0x186 github.com/luci/luci-go/common/system/ctxcmd.(*CtxCmd).Start.func1(0x11810b40, 0x11810b00, 0x11769050) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/system/ctxcmd/proc.go:120 +0x81 created by github.com/luci/luci-go/common/system/ctxcmd.(*CtxCmd).Start E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/system/ctxcmd/proc.go:122 +0x11d goroutine 53 [select, 195 minutes]: github.com/luci/luci-go/common/system/ctxcmd.(*CtxCmd).Start.func2(0x11810b40, 0x11810b80, 0x11769050, 0x331e4c58, 0x11806ec0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/system/ctxcmd/proc.go:142 +0x2d4 created by github.com/luci/luci-go/common/system/ctxcmd.(*CtxCmd).Start E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/system/ctxcmd/proc.go:165 +0x17a goroutine 45 [select, 195 minutes]: github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2(0x11765700, 0x118695d0, 0x118a0040, 0x117d5260) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:456 +0x1ef created by github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:468 +0x1c8 goroutine 9 [select, 195 minutes]: github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2(0x11765700, 0x11774ba0, 0x118a4120, 0x117d5260) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:456 +0x1ef created by github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:468 +0x1c8 goroutine 46 [select]: gopkg.in/natefinch/npipe%2ev2.(*PipeConn).completeRequest(0x11806f60, 0x0, 0x290030, 0x590e2b10, 0x0, 0x590eaa80, 0x290030, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:415 +0x397 gopkg.in/natefinch/npipe%2ev2.(*PipeConn).Read(0x11806f60, 0x42afb000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:442 +0x112 github.com/luci/luci-go/common/iotools.(*DeadlineReader).Read(0x11803c60, 0x42afb000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/iotools/deadlinereader.go:50 +0x23b github.com/luci/luci-go/logdog/client/butler.(*stream).readChunk(0x118a0040, 0xad1600) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/stream.go:33 +0xe1 github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2.2(0x11765880, 0x118a0040) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:450 +0x43 created by github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2 E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:452 +0xad goroutine 10 [select, 195 minutes]: gopkg.in/natefinch/npipe%2ev2.(*PipeConn).completeRequest(0x11807020, 0x0, 0x290030, 0x118dc254, 0x0, 0x118de5a0, 0x290030, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:415 +0x397 gopkg.in/natefinch/npipe%2ev2.(*PipeConn).Read(0x11807020, 0x11875000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:442 +0x112 github.com/luci/luci-go/common/iotools.(*DeadlineReader).Read(0x11803ca0, 0x11875000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/iotools/deadlinereader.go:50 +0x23b github.com/luci/luci-go/logdog/client/butler.(*stream).readChunk(0x118a4120, 0xad1600) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/stream.go:33 +0xe1 github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2.2(0x1168bd00, 0x118a4120) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:450 +0x43 created by github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2 E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:452 +0xad goroutine 92 [runnable]: gopkg.in/natefinch/npipe%2ev2.(*PipeConn).completeRequest(0x118b3400, 0x0, 0x290030, 0x6445d6e4, 0x0, 0x64478de0, 0x290030, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:415 +0x397 gopkg.in/natefinch/npipe%2ev2.(*PipeConn).Read(0x118b3400, 0x518d8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:442 +0x112 github.com/luci/luci-go/common/iotools.(*DeadlineReader).Read(0x118b0700, 0x518d8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/iotools/deadlinereader.go:50 +0x23b github.com/luci/luci-go/logdog/client/butler.(*stream).readChunk(0x118a0d60, 0xad1600) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/stream.go:33 +0xe1 github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2.2(0x118f2240, 0x118a0d60) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:450 +0x43 created by github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2 E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:452 +0xad goroutine 101 [syscall, 195 minutes, locked to thread]: syscall.Syscall(0x75dd1136, 0x2, 0x3c4, 0xffffffff, 0x0, 0x0, 0x1, 0x118daf8c) E:/b/rr/tmpidromi/w/golang/go/src/runtime/syscall_windows.go:163 +0x4a syscall.WaitForSingleObject(0x3c4, 0xffffffff, 0x410988, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/syscall/zsyscall_windows.go:693 +0x5f gopkg.in/natefinch/npipe%2ev2.waitForCompletion(0xf4, 0x118de5a0, 0x118dafb8, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:182 +0x3a gopkg.in/natefinch/npipe%2ev2.(*PipeConn).completeRequest.func1(0x11807020, 0x118de5a0, 0x118e6280) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:412 +0x2b created by gopkg.in/natefinch/npipe%2ev2.(*PipeConn).completeRequest E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:414 +0x168 goroutine 91 [select, 195 minutes]: github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2(0x11765700, 0x11869d10, 0x118a0d60, 0x117d5260) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:456 +0x1ef created by github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:468 +0x1c8 goroutine 461 [chan receive]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x11869420, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 2037 [chan receive]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x11868680, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 101998 [chan receive]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x11bb63e0, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 1154779 [syscall, locked to thread]: syscall.Syscall(0x75dd1136, 0x2, 0x524, 0xffffffff, 0x0, 0x0, 0x1, 0x15e52f8c) E:/b/rr/tmpidromi/w/golang/go/src/runtime/syscall_windows.go:163 +0x4a syscall.WaitForSingleObject(0x524, 0xffffffff, 0x410988, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/syscall/zsyscall_windows.go:693 +0x5f gopkg.in/natefinch/npipe%2ev2.waitForCompletion(0x528, 0x590f3680, 0x15e52fb8, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:182 +0x3a gopkg.in/natefinch/npipe%2ev2.(*PipeConn).completeRequest.func1(0x590f2c00, 0x590f3680, 0x5b7ba200) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:412 +0x2b created by gopkg.in/natefinch/npipe%2ev2.(*PipeConn).completeRequest E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:414 +0x168 goroutine 680 [chan receive]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x1198d6e0, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 295577 [chan receive, 2 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x14059d50, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 412023 [chan receive, 2 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x1236ec40, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 1154777 [select]: github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2(0x11765700, 0x5b7bc4f0, 0x590f3580, 0x117d5260) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:456 +0x1ef created by github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:468 +0x1c8 goroutine 233395 [chan receive, 2 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x139234e0, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 1154712 [select]: gopkg.in/natefinch/npipe%2ev2.(*PipeConn).completeRequest(0x590f2c00, 0x0, 0x290030, 0x5b7bc540, 0x0, 0x590f3680, 0x290030, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:415 +0x397 gopkg.in/natefinch/npipe%2ev2.(*PipeConn).Read(0x590f2c00, 0x2ab0c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:442 +0x112 github.com/luci/luci-go/common/iotools.(*DeadlineReader).Read(0x590cf030, 0x2ab0c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/iotools/deadlinereader.go:50 +0x23b github.com/luci/luci-go/logdog/client/butler.(*stream).readChunk(0x5907bbe0, 0xad1600) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/stream.go:33 +0xe1 github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2.2(0x590508c0, 0x5907bbe0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:450 +0x43 created by github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2 E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:452 +0xad goroutine 222320 [chan receive, 1 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x13f374b0, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 245615 [chan receive, 2 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x1393c580, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 255889 [chan receive, 2 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x13f99000, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 177485 [chan receive]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x136a1dc0, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 190912 [chan receive, 1 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x1425adc0, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 1154757 [syscall, locked to thread]: syscall.Syscall(0x75dd1136, 0x2, 0x51c, 0xffffffff, 0x0, 0x0, 0x1, 0x2ebe1f8c) E:/b/rr/tmpidromi/w/golang/go/src/runtime/syscall_windows.go:163 +0x4a syscall.WaitForSingleObject(0x51c, 0xffffffff, 0x410988, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/syscall/zsyscall_windows.go:693 +0x5f gopkg.in/natefinch/npipe%2ev2.waitForCompletion(0xe4, 0x590eaa80, 0x2ebe1fb8, 0x0, 0x0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:182 +0x3a gopkg.in/natefinch/npipe%2ev2.(*PipeConn).completeRequest.func1(0x11806f60, 0x590eaa80, 0x58ed1ac0) E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:412 +0x2b created by gopkg.in/natefinch/npipe%2ev2.(*PipeConn).completeRequest E:/b/rr/tmpidromi/w/infra/go/.vendor/src/gopkg.in/natefinch/npipe.v2/npipe_windows.go:414 +0x168 goroutine 499782 [chan receive]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x1a65a5d0, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 350520 [chan receive, 2 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x123a5090, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 498182 [IO wait]: net.runtime_pollWait(0x331e60a8, 0x72, 0x6299d1ec) E:/b/rr/tmpidromi/w/golang/go/src/runtime/netpoll.go:160 +0x55 net.(*pollDesc).Wait(0x19336370, 0x72, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/net/fd_poll_runtime.go:73 +0x35 net.(*ioSrv).ExecIO(0x1176c890, 0x193362d8, 0x9a52f0, 0x7, 0xad13c4, 0x8, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/net/fd_windows.go:183 +0x122 net.(*netFD).Read(0x193362a0, 0x1538c000, 0x400, 0x400, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/net/fd_windows.go:482 +0x129 net.(*conn).Read(0x138732d8, 0x1538c000, 0x400, 0x400, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/net/net.go:172 +0xb9 crypto/tls.(*block).readFromUntil(0x16d2d6e0, 0x331e6198, 0x138732d8, 0x5, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/crypto/tls/conn.go:460 +0xa4 crypto/tls.(*Conn).readRecord(0x165321c0, 0xad1717, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/crypto/tls/conn.go:562 +0x269 crypto/tls.(*Conn).Read(0x165321c0, 0x19bc3000, 0x1000, 0x1000, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/crypto/tls/conn.go:939 +0x130 bufio.(*Reader).fill(0x13463110) E:/b/rr/tmpidromi/w/golang/go/src/bufio/bufio.go:97 +0x172 bufio.(*Reader).Read(0x13463110, 0x1249c560, 0x9, 0x9, 0x16532368, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/bufio/bufio.go:207 +0x1b9 io.ReadAtLeast(0x331e64d0, 0x13463110, 0x1249c560, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/io/io.go:297 +0xab io.ReadFull(0x331e64d0, 0x13463110, 0x1249c560, 0x9, 0x9, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/io/io.go:315 +0x51 net/http.http2readFrameHeader(0x1249c560, 0x9, 0x9, 0x331e64d0, 0x13463110, 0x0, 0x0, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/net/http/h2_bundle.go:715 +0x7b net/http.(*http2Framer).ReadFrame(0x1249c540, 0x0, 0x0, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/net/http/h2_bundle.go:906 +0xc4 net/http.(*http2clientConnReadLoop).run(0x14fb7440, 0x0, 0x0) E:/b/rr/tmpidromi/w/golang/go/src/net/http/h2_bundle.go:5583 +0xaa net/http.(*http2ClientConn).readLoop(0x19336b60) E:/b/rr/tmpidromi/w/golang/go/src/net/http/h2_bundle.go:5544 +0x139 created by net/http.(*http2Transport).NewClientConn E:/b/rr/tmpidromi/w/golang/go/src/net/http/h2_bundle.go:4975 +0xb9a goroutine 1217864 [runnable]: github.com/luci/luci-go/common/sync/cancelcond.(*Cond).Wait.func1(0x644704c0, 0x33710140, 0x64478540, 0x63ba17f8, 0x1176dce0, 0x64470480) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/cancelcond/cancelCond.go:54 +0x13f created by github.com/luci/luci-go/common/sync/cancelcond.(*Cond).Wait E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/cancelcond/cancelCond.go:63 +0x156 goroutine 212715 [chan receive, 1 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x137397c0, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 202966 [chan receive, 1 minutes]: github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody.func1(0x1175f760, 0x13e78080, 0x11765701, 0x11863cc0) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:139 +0x98 created by github.com/luci/luci-go/common/sync/parallel.(*Runner).dispatchLoopBody E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/common/sync/parallel/runner.go:143 +0xed goroutine 1154711 [select]: github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams.func2(0x11765700, 0x590659a0, 0x5907bbe0, 0x117d5260) E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:456 +0x1ef created by github.com/luci/luci-go/logdog/client/butler.(*Butler).runStreams E:/b/rr/tmpidromi/w/infra/go/src/github.com/luci/luci-go/logdog/client/butler/butler.go:468 +0x1c8
,
Dec 6 2016
TBH the thing to do is probably capture the BuildBot STDOUT and run it locally on Butler w/ memory profiling enabled. In theory, this shouldn't be a problem. LogDog retains one entry per log stream (not too large), so a few thousand entries here, and buffers data for up to 30s from each log stream. As it sends data off, it should recycle those buffers, and it uses a sync.Pool, which will naturally expire old buffers. It's a streaming system, so all of the data should be in, process, out. The two options here are: 1) Something is retaining memory where it shouldn't be, or 2) The linearly-accumulating stream data is consuming too much memory. (2) seems unlikely b/c what's a few hundred thousand bytes, and (1) should not happen b/c of the aforementioned streaming design. A third option is that the Go runtime is not cool with something here, and that upgrading to a newer runtime (which I'm actually doing this morning) will fix it. If this is an emergency, we can disable LogDog for this builder. I am not going to have time to look at it for at least a few hours, so punting back to you for a decision.
,
Dec 6 2016
It looks like logdog is actually OOM-ing. See http://shortn/_lOLQdKcygD. Could it be that we are just sending too many logs through logdog. The log stream in question has about 120,00 log lines, each of which are pretty small. This is affecting more than 1 builder, so I don't think we want to disable it yet... I think it could be (2)? The log data is about 12 MB. The data that gets sent before this is about 5 MB. FWIW, I'm on another bot, and I see this line (by running top) 2949 chrome-+ 20 0 1655744 977.1m 7000 S 0.0 3.2 1:55.17 logdog_butler So logdog is using about 1 GB of memory. I'm going to see if there's any way to reproduce this locally
,
Dec 6 2016
This may be fixed by the latest version of logdog. I was not able to reproduce locally with a version I built from head. dnj@ is going to push out a new version of logdog soon. I'll check this tomorrow to see if we see the error any more.
,
Dec 6 2016
The following revision refers to this bug: https://chromium.googlesource.com/external/github.com/luci/luci-go.git/+/f5e96b94465d190bb8022c900506e17e7c5e1a70 commit f5e96b94465d190bb8022c900506e17e7c5e1a70 Author: dnj <dnj@chromium.org> Date: Tue Dec 06 20:49:47 2016 Add profiling, integrate into Butler/Annotee. Add a profiling package to run a profiling HTTP server and allow periodic profiling metric collection. Integrate this into LogDog's Annotee and Butler programs. BUG= chromium:671616 TEST=local - Ran Butler locally, bound to HTTP server, seems to work. Review-Url: https://codereview.chromium.org/2548253004 [add] https://crrev.com/f5e96b94465d190bb8022c900506e17e7c5e1a70/common/runtime/profiling/profiler.go [modify] https://crrev.com/f5e96b94465d190bb8022c900506e17e7c5e1a70/logdog/client/cmd/logdog_annotee/main.go [modify] https://crrev.com/f5e96b94465d190bb8022c900506e17e7c5e1a70/logdog/client/cmd/logdog_butler/main.go
,
Dec 6 2016
The following revision refers to this bug: https://chromium.googlesource.com/infra/infra.git/+/25755a2c316937ee44a6432163dc5e2f9c85cf58 commit 25755a2c316937ee44a6432163dc5e2f9c85cf58 Author: Dan Jacques <dnj@chromium.org> Date: Tue Dec 06 20:57:31 2016 Roll luci-go. infra/go/src/github.com/luci/luci-go: f5e96b9 Add profiling, integrate into Butler/Annotee. 01914a3 Add GAE datastore caching layer implementation. TBR=martiniss@chromium.org BUG= chromium:671616 TEST=None Change-Id: I0bc415d20f2a736d3b596d59a0fa19e6eb610de6 Reviewed-on: https://chromium-review.googlesource.com/417094 Reviewed-by: Daniel Jacques <dnj@chromium.org> Commit-Queue: Daniel Jacques <dnj@chromium.org> [modify] https://crrev.com/25755a2c316937ee44a6432163dc5e2f9c85cf58/DEPS
,
Dec 7 2016
It looks like this has been fixed with the recent luci-go roll!
,
Mar 4 2017
,
Apr 17 2017
,
May 30 2017
,
Aug 1 2017
,
Oct 14 2017
|
||||||||
►
Sign in to add a comment |
||||||||
Comment 1 by martiniss@chromium.org
, Dec 6 2016Owner: d...@chromium.org
Status: Assigned (was: Available)