Browser process is leaking handles [was: OOH (out-of-handles) when creating SchedulerWorker] |
||||||
Issue descriptionreporter:emaxx@google.com Magic Signature: base::internal::SchedulerWorker::SchedulerWorker Crash link: https://crash.corp.google.com/browse?q=product.name%3D%27Chrome%27+AND+product.version%3D%2768.0.3425.0%27+AND+expanded_custom_data.ChromeCrashProto.ptype%3D%27browser%27+AND+expanded_custom_data.ChromeCrashProto.magic_signature_1.name%3D%27base%3A%3Ainternal%3A%3ASchedulerWorker%3A%3ASchedulerWorker%27&stbtiq=&reportid=&index=0 ------------------------------------------------------------------------------- Sample Report ------------------------------------------------------------------------------- Product name: Chrome Magic Signature : base::internal::SchedulerWorker::SchedulerWorker Product Version: 68.0.3425.0 Process type: browser Report ID: e224cef77bfc1514 Report Url: https://crash.corp.google.com/e224cef77bfc1514 Report Time: 2018-05-09T12:54:05-07:00 Upload Time: 2018-05-09T12:54:11.259-07:00 Uptime: 7923000 ms OS Name: Windows NT OS Version: 10.0.16299 431 CPU Architecture: amd64 CPU Info: family 6 model 158 stepping 9 ------------------------------------------------------------------------------- Crashing thread: Thread index: 0. Stack Quality: 100%. Thread id: 9620. ------------------------------------------------------------------------------- 0x00007fffe1e2f5e9 (chrome.dll - waitable_event_win.cc: 30) base::WaitableEvent::WaitableEvent(base::WaitableEvent::ResetPolicy,base::WaitableEvent::InitialState) 0x00007fffe3012637 (chrome.dll - scheduler_worker.cc: 54) base::internal::SchedulerWorker::SchedulerWorker(base::ThreadPriority,std::unique_ptr<base::internal::SchedulerWorker::Delegate,std::default_delete<base::internal::SchedulerWorker::Delegate> >,base::internal::TrackedRef<base::internal::TaskTracker>,base::internal::SchedulerLock const *,base::SchedulerBackwardCompatibility) 0x00007fffe1ebc253 (chrome.dll - scheduler_worker_pool_impl.cc: 850) base::internal::SchedulerWorkerPoolImpl::CreateRegisterAndStartSchedulerWorkerLockRequired() 0x00007fffe1e4acf1 (chrome.dll - scheduler_worker_pool_impl.cc: 787) base::internal::SchedulerWorkerPoolImpl::WakeUpOneWorkerLockRequired() 0x00007fffe1e4a71c (chrome.dll - scheduler_worker_pool_impl.cc: 272) base::internal::SchedulerWorkerPoolImpl::OnCanScheduleSequence(scoped_refptr<base::internal::Sequence>) 0x00007fffe1e4a018 (chrome.dll - scheduler_worker_pool.cc: 214) base::internal::SchedulerWorkerPool::PostTaskWithSequenceNow(base::internal::Task,scoped_refptr<base::internal::Sequence>) 0x00007fffe1e49ca5 (chrome.dll - scheduler_worker_pool.cc: 152) base::internal::SchedulerWorkerPool::PostTaskWithSequence(base::internal::Task,scoped_refptr<base::internal::Sequence>) 0x00007fffe1e499a2 (chrome.dll - scheduler_worker_pool.cc: 105) base::internal::SchedulerSequencedTaskRunner::PostDelayedTask(base::Location const &,base::OnceCallback<void >,base::TimeDelta) 0x00007fffe1e30f5c (chrome.dll - task_runner.cc: 44) base::TaskRunner::PostTask(base::Location const &,base::OnceCallback<void >) 0x00007fffe2e64f33 (chrome.dll - storage_frontend.cc: 149) extensions::StorageFrontend::RunWithStorage(scoped_refptr<extensions::Extension const >,extensions::settings_namespace::Namespace,base::RepeatingCallback<void > const &) 0x00007fffe2e63e02 (chrome.dll - storage_api.cc: 61) extensions::SettingsFunction::Run() 0x00007fffe2d72dd7 (chrome.dll - extension_function.cc: 451) ExtensionFunction::RunWithValidation() 0x00007fffe2d74344 (chrome.dll - extension_function_dispatcher.cc: 486) extensions::ExtensionFunctionDispatcher::DispatchWithCallbackInternal(ExtensionHostMsg_Request_Params const &,content::RenderFrameHost *,int,base::RepeatingCallback<void > const &) 0x00007fffe2d73f47 (chrome.dll - extension_function_dispatcher.cc: 380) extensions::ExtensionFunctionDispatcher::Dispatch(ExtensionHostMsg_Request_Params const &,content::RenderFrameHost *,int) 0x00007fffe2d8a39a (chrome.dll - ipc_message_templates.h: 146) IPC::MessageT<ExtensionHostMsg_Request_Meta,std::tuple<ExtensionHostMsg_Request_Params>,void>::Dispatch<extensions::ExtensionWebContentsObserver,extensions::ExtensionWebContentsObserver,content::RenderFrameHost,void (extensions::ExtensionWebContentsObserver::*)(content::RenderFrameHost *, const ExtensionHostMsg_Request_Params &)> 0x00007fffe248267f (chrome.dll - extension_web_contents_observer.cc: 225) extensions::ExtensionWebContentsObserver::OnMessageReceived(IPC::Message const &,content::RenderFrameHost *) 0x00007fffe24825c6 (chrome.dll - chrome_extension_web_contents_observer.cc: 106) extensions::ChromeExtensionWebContentsObserver::OnMessageReceived(IPC::Message const &,content::RenderFrameHost *) 0x00007fffe2481c20 (chrome.dll - web_contents_impl.cc: 790) content::WebContentsImpl::OnMessageReceived(content::RenderFrameHostImpl *,IPC::Message const &) 0x00007fffe2480b23 (chrome.dll - render_frame_host_impl.cc: 924) content::RenderFrameHostImpl::OnMessageReceived(IPC::Message const &) 0x00007fffe2480974 (chrome.dll - render_process_host_impl.cc: 3027) content::RenderProcessHostImpl::OnMessageReceived(IPC::Message const &) 0x00007fffe248085a (chrome.dll - ipc_channel_proxy.cc: 320) IPC::ChannelProxy::Context::OnDispatchMessage(IPC::Message const &) 0x00007fffe1e36034 (chrome.dll - task_annotator.cc: 101) base::debug::TaskAnnotator::RunTask(char const *,base::PendingTask *) 0x00007fffe1e35adb (chrome.dll - message_loop.cc: 319) base::MessageLoop::RunTask(base::PendingTask *) 0x00007fffe1e35527 (chrome.dll - message_loop.cc: 373) base::MessageLoop::DoWork() 0x00007fffe1f42b08 (chrome.dll - message_pump_win.cc: 173) base::MessagePumpForUI::DoRunLoop() 0x00007fffe1e7fe67 (chrome.dll - message_pump_win.cc: 56) base::MessagePumpWin::Run(base::MessagePump::Delegate *) 0x00007fffe1e350a0 (chrome.dll - run_loop.cc: 131) base::RunLoop::Run() 0x00007fffe2204810 (chrome.dll - chrome_browser_main.cc: 2137) ChromeBrowserMainParts::MainMessageLoopRun(int *) 0x00007fffe2204607 (chrome.dll - browser_main_loop.cc: 976) content::BrowserMainLoop::RunMainMessageLoopParts() 0x00007fffe22045b2 (chrome.dll - browser_main_runner.cc: 160) content::BrowserMainRunnerImpl::Run() 0x00007fffe1e29c39 (chrome.dll - browser_main.cc: 46) content::BrowserMain(content::MainFunctionParams const &) 0x00007fffe1e29ad6 (chrome.dll - content_main_runner.cc: 640) content::RunNamedProcessTypeMain(std::basic_string<char,std::char_traits<char>,std::allocator<char> > const &,content::MainFunctionParams const &,content::ContentMainDelegate *) 0x00007fffe1e29980 (chrome.dll - content_main_runner.cc: 943) content::ContentMainRunnerImpl::Run() 0x00007fffe1e1569a (chrome.dll - main.cc: 452) service_manager::Main(service_manager::MainParams const &) 0x00007fffe1e15167 (chrome.dll - content_main.cc: 19) content::ContentMain(content::ContentMainParams const &) 0x00007fffe1e11c01 (chrome.dll - chrome_main.cc: 101) ChromeMain 0x00007ff765df350b (chrome.exe - main_dll_loader_win.cc: 201) MainDllLoader::Launch(HINSTANCE__ *,base::TimeTicks) 0x00007ff765df1698 (chrome.exe - chrome_exe_main_win.cc: 230) wWinMain 0x00007ff765ecccc2 (chrome.exe - exe_common.inl: 283) __scrt_common_main_seh 0x00007ff819ab1fe3 (KERNEL32.DLL + 0x00011fe3) ------------------------------------------------------------------------------- Manual regression range finder link ------------------------------------------------------------------------------- https://crash.corp.google.com/browse?q=expanded_custom_data.ChromeCrashProto.magic_signature_1.name%3D%27base%3A%3Ainternal%3A%3ASchedulerWorker%3A%3ASchedulerWorker%27+AND+expanded_custom_data.ChromeCrashProto.ptype%3D%27browser%27#-property-selector,-samplereports,+productname,+productversion:1000,+directory,-clientid,+operatingsystem,+url,+simplifiedurl,+extensions
,
May 10 2018
Analyzed a minidump in WinDBG. The crash is caused by a CHECK() that verifies whether the handle returned by CreateEvent() is valid https://cs.chromium.org/chromium/src/base/synchronization/waitable_event_win.cc?l=30&rcl=1bce37bb0f8e63c54137b5e0e31e50fc5b0daf86. !gle -all prints: LastErrorValue: (Win32) 0 (0) - The operation completed successfully. LastStatusValue: (NTSTATUS) 0xc000009a - Insufficient system resources exist to complete the API. Is it possible that there is code in Chrome that creates a lot of handles and doesn't free them?
,
May 10 2018
Will try to augment dumps with handleCount info in breadcrumbs so that we can get an idea of whether this is the problem (and then assess a broader solution to get a handle on handles..!) FTR, fdoray found that 16 million handles is the cap in a process so that sounds a bit crazy..! https://blogs.technet.microsoft.com/markrussinovich/2009/09/29/pushing-the-limits-of-windows-handles/
,
May 10 2018
Looking into the Handles column in Process Hacker for my Canary it looks like we have a handle leak in CreateProcess. My browser process was at a crazy 250,000 handle count at 12pm and 3 hours later it's now at 267,000 and still going up.
"!handle 0 7" yields :
Handle 105ff8 ( **many like these** )
Type Process
Attributes 0
GrantedAccess 0x1fffff:
Delete,ReadControl,WriteDac,WriteOwner,Synch
Terminate,CreateThread,,VMOp,VMRead,VMWrite,DupHandle,CreateProcess,SetQuota,SetInfo,QueryInfo,SetPort
HandleCount 702
PointerCount 372871
Name <none>
(...)
266937 Handles
Type Count
None 6
Event 1550
Section 476
File 827
Directory 2
Mutant 63
WindowStation 3
Semaphore 32
Key 136
Token 93
Process 262875
Thread 114
Desktop 2
IoCompletion 8
Timer 1
Job 80
TpWorkerFactory 3
ALPC Port 18
WaitCompletionPacket 648
,
May 14 2018
This looks to be a mojo leak, the Process handle count tracks exactly with calls to chrome!mojo::edk::Channel::Message::RewriteHandles/mojo::edk::ScopedProcessHandle::CloneFrom. I suspect https://chromium-review.googlesource.com/c/chromium/src/+/1036459. Ken PTAL. Typical trace from bu chrome!mojo::edk::ScopedProcessHandle::CloneFrom "!handle 0 0 Process; kc; g" 4822 handles of type Process # Call Site 00 chrome!mojo::edk::ScopedProcessHandle::CloneFrom 01 chrome!mojo::edk::Channel::Message::RewriteHandles 02 chrome!mojo::edk::NodeChannel::WriteChannelMessage 03 chrome!mojo::edk::NodeChannel::SendChannelMessage 04 chrome!mojo::edk::NodeController::SendPeerEvent 05 chrome!mojo::edk::NodeController::ForwardEvent 06 chrome!mojo::edk::ports::Node::SendUserMessageInternal 07 chrome!mojo::edk::ports::Node::SendUserMessage 08 chrome!mojo::edk::NodeController::SendUserMessage 09 chrome!mojo::edk::MessagePipeDispatcher::WriteMessage 0a chrome!mojo::edk::Core::WriteMessage 0b chrome!mojo::WriteMessageNew 0c chrome!mojo::Connector::Accept 0d chrome!network::mojom::URLLoaderClientProxy::OnStartLoadingResponseBody 0e chrome!content::MojoAsyncResourceHandler::OnReadCompleted 0f chrome!content::InterceptingResourceHandler::OnReadCompleted 10 chrome!content::LayeredResourceHandler::OnReadCompleted 11 chrome!std::_Compressed_pair<std::default_delete<content::ResourceController>,content::ResourceController *,1>::_Compressed_pair 12 chrome!std::_Unique_ptr_base<content::ResourceController,std::default_delete<content::ResourceController> >::_Unique_ptr_base 13 chrome!std::unique_ptr<content::ResourceController,std::default_delete<content::ResourceController> >::unique_ptr 14 chrome!content::CrossSiteDocumentResourceHandler::OnReadCompleted 15 chrome!content::MimeSniffingResourceHandler::OnReadCompleted 16 chrome!content::LayeredResourceHandler::OnReadCompleted 17 chrome!content::ResourceLoader::CompleteRead 18 chrome!content::ResourceLoader::OnReadCompleted 19 chrome!content::ResourceLoader::PrepareToReadMore 1a chrome!content::ResourceLoader::CompleteResponseStarted 1b chrome!content::ResourceLoader::OnResponseStarted 1c chrome!net::URLRequestJob::NotifyHeadersComplete 1d chrome!net::URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete 1e chrome!net::URLRequestHttpJob::OnStartCompleted 1f chrome!base::RepeatingCallback<void (int)>::Run 20 chrome!net::HttpCache::Transaction::DoLoop 21 chrome!base::RepeatingCallback<void (int)>::Run 22 chrome!net::HttpNetworkTransaction::DoCallback 23 chrome!base::OnceCallback<void (int)>::Run 24 chrome!net::QuicHttpStream::DoCallback 25 chrome!base::RepeatingCallback<void (int)>::Run 26 chrome!net::QuicChromiumClientStream::Handle::ResetAndRun 27 chrome!net::QuicChromiumClientStream::Handle::OnInitialHeadersAvailable 28 chrome!base::OnceCallback<void ()>::Run 29 chrome!base::debug::TaskAnnotator::RunTask 2a chrome!base::MessageLoop::RunTask 2b chrome!base::MessageLoop::DeferOrRunPendingTask 2c chrome!base::MessageLoop::DoWork 2d chrome!base::MessagePumpForIO::DoRunLoop 2e chrome!base::MessagePumpWin::Run 2f chrome!base::RunLoop::Run 30 chrome!content::BrowserProcessSubThread::IOThreadRun 31 chrome!base::Thread::ThreadMain 32 chrome!base::`anonymous namespace'::ThreadFunc 33 KERNEL32!BaseThreadInitThunk 34 ntdll!RtlUserThreadStart 4823 handles of type Process # Call Site 00 chrome!mojo::edk::ScopedProcessHandle::CloneFrom 01 chrome!mojo::edk::Channel::Message::RewriteHandles 02 chrome!mojo::edk::NodeChannel::WriteChannelMessage 03 chrome!mojo::edk::NodeChannel::SendChannelMessage 04 chrome!mojo::edk::NodeController::SendPeerEvent 05 chrome!mojo::edk::NodeController::ForwardEvent 06 chrome!mojo::edk::ports::Node::SendUserMessageInternal 07 chrome!mojo::edk::ports::Node::SendUserMessage 08 chrome!mojo::edk::NodeController::SendUserMessage 09 chrome!mojo::edk::MessagePipeDispatcher::WriteMessage 0a chrome!mojo::edk::Core::WriteMessage 0b chrome!mojo::WriteMessageNew 0c chrome!mojo::Connector::Accept 0d chrome!network::mojom::URLLoaderClientProxy::OnStartLoadingResponseBody 0e chrome!content::MojoAsyncResourceHandler::OnReadCompleted 0f chrome!content::InterceptingResourceHandler::OnReadCompleted 10 chrome!content::LayeredResourceHandler::OnReadCompleted 11 chrome!std::_Compressed_pair<std::default_delete<content::ResourceController>,content::ResourceController *,1>::_Compressed_pair 12 chrome!std::_Unique_ptr_base<content::ResourceController,std::default_delete<content::ResourceController> >::_Unique_ptr_base 13 chrome!std::unique_ptr<content::ResourceController,std::default_delete<content::ResourceController> >::unique_ptr 14 chrome!content::CrossSiteDocumentResourceHandler::OnReadCompleted 15 chrome!content::MimeSniffingResourceHandler::OnReadCompleted 16 chrome!content::LayeredResourceHandler::OnReadCompleted 17 chrome!content::ResourceLoader::CompleteRead 18 chrome!content::ResourceLoader::OnReadCompleted 19 chrome!base::RepeatingCallback<void (int)>::Run 1a chrome!net::FilterSourceStream::OnIOComplete 1b chrome!base::RepeatingCallback<void (int)>::Run 1c chrome!net::URLRequestJob::ReadRawDataComplete 1d chrome!net::URLRequestHttpJob::OnReadCompleted 1e chrome!base::RepeatingCallback<void (int)>::Run 1f chrome!net::HttpCache::Transaction::DoLoop 20 chrome!base::RepeatingCallback<void (int)>::Run 21 chrome!net::HttpNetworkTransaction::DoCallback 22 chrome!base::OnceCallback<void (int)>::Run 23 chrome!net::SpdyHttpStream::DoResponseCallback 24 chrome!net::SpdyHttpStream::OnClose 25 chrome!net::SpdyStream::OnClose 26 chrome!net::SpdySession::DeleteStream 27 chrome!net::SpdySession::CloseActiveStreamIterator 28 chrome!net::SpdyStream::OnDataReceived 29 chrome!net::SpdySession::OnStreamEnd 2a chrome!net::QuicHttpDecoderAdapter::OnDataEnd 2b chrome!http2::DataPayloadDecoder::ResumeDecodingPayload 2c chrome!http2::Http2FrameDecoder::StartDecodingDataPayload 2d chrome!http2::Http2FrameDecoder::StartDecodingPayload 2e chrome!http2::Http2DecoderAdapter::ProcessInputFrame 2f chrome!http2::Http2DecoderAdapter::ProcessInput 30 chrome!net::SpdySession::DoReadComplete 31 chrome!net::SpdySession::DoReadLoop 32 chrome!base::OnceCallback<void (int)>::Run 33 chrome!net::SSLClientSocketImpl::DoReadCallback 34 chrome!net::SSLClientSocketImpl::RetryAllOperations 35 chrome!base::OnceCallback<void (int)>::Run 36 chrome!net::TCPClientSocket::DidCompleteReadWrite 37 chrome!net::TCPClientSocket::DidCompleteRead 38 chrome!base::internal::FunctorTraits<void (net::SOCKSClientSocket::*)(base::OnceCallback<void (int)>, int),void>::Invoke 39 chrome!base::internal::InvokeHelper<0,void>::MakeItSo 3a chrome!base::internal::Invoker<base::internal::BindState<void (net::SOCKSClientSocket::*)(base::OnceCallback<void (int)>, int),base::internal::UnretainedWrapper<net::SOCKSClientSocket>,base::OnceCallback<void (int)> >,void (int)>::RunImpl 3b chrome!base::internal::Invoker<base::internal::BindState<void (net::SOCKSClientSocket::*)(base::OnceCallback<void (int)>, int),base::internal::UnretainedWrapper<net::SOCKSClientSocket>,base::OnceCallback<void (int)> >,void (int)>::RunOnce 3c chrome!base::OnceCallback<void (int)>::Run 3d chrome!net::TCPSocketWin::DidSignalRead 3e chrome!base::OnceCallback<void ()>::Run 3f chrome!base::debug::TaskAnnotator::RunTask 40 chrome!base::MessageLoop::RunTask 41 chrome!base::MessageLoop::DeferOrRunPendingTask 42 chrome!base::MessageLoop::DoWork 43 chrome!base::MessagePumpForIO::DoRunLoop 44 chrome!base::MessagePumpWin::Run 45 chrome!base::RunLoop::Run 46 chrome!content::BrowserProcessSubThread::IOThreadRun 47 chrome!base::Thread::ThreadMain 48 chrome!base::`anonymous namespace'::ThreadFunc 49 KERNEL32!BaseThreadInitThunk 4a ntdll!RtlUserThreadStart (8120.8c74): Break instruction exception - code 80000003 (first chance) ntdll!DbgBreakPoint: 00007ffb`01669920 cc int 3
,
May 14 2018
Etienne might be interested in adding handle leak detection to slow reports. CCing him onto this bug.
,
May 14 2018
Thanks, this certainly looks like my fault. D: Will take a look ASAP.
,
May 14 2018
BTW this didn't make it into any branches AFAICT so a simple revert should suffice.
,
May 14 2018
,
May 14 2018
,
May 16 2018
The following revision refers to this bug: https://chromium.googlesource.com/chromium/src.git/+/8c876c84608a5beafe7d24fa33eb898cb32a8101 commit 8c876c84608a5beafe7d24fa33eb898cb32a8101 Author: Gabriel Charette <gab@chromium.org> Date: Wed May 16 15:07:00 2018 [Crash Reports] Add HandleCount to system and process state on Windows. We recently found a major handle leak. Adding the handle count as a field will help identify spurious crashes that were caused by this and avoid 100 owners looking into the same thing and cursing Windows + crash. R=siggi@chromium.org Bug: 841565 Change-Id: Ife9126450397df0bb736860141a28888143062f2 Reviewed-on: https://chromium-review.googlesource.com/1053806 Reviewed-by: Sigurður Ásgeirsson <siggi@chromium.org> Commit-Queue: Gabriel Charette <gab@chromium.org> Cr-Commit-Position: refs/heads/master@{#559099} [modify] https://crrev.com/8c876c84608a5beafe7d24fa33eb898cb32a8101/components/browser_watcher/dump_stability_report_main_win.cc [modify] https://crrev.com/8c876c84608a5beafe7d24fa33eb898cb32a8101/components/browser_watcher/stability_report.proto [modify] https://crrev.com/8c876c84608a5beafe7d24fa33eb898cb32a8101/components/browser_watcher/stability_report_user_stream_data_source.cc |
||||||
►
Sign in to add a comment |
||||||
Comment 1 by emaxx@chromium.org
, May 9 2018Owner: gab@chromium.org
Status: Assigned (was: Untriaged)