Issue metadata
Sign in to add a comment
|
"Problem when trying to gather stack trace" during WebGL test failures on Android |
||||||||||||||||||||
Issue descriptionSee issue 912161 for spotted occurrences. Not sure if related to recent enabling of Crashpad or some other problem. Since most of these stacks are in catapult, assigning to John for triage, but would appreciate if Ken and Joshua could also take a look. Example log of the problem: Timed out. Dumping threads. ******************************************************************************** Stack dump for thread 'TimeoutThread-1-for-MainThread' ******************************************************************************** File: "/b/swarming/w/ir/.swarming_module/lib/python2.7/threading.py", line 778, in __bootstrap raise File: "/b/swarming/w/ir/.swarming_module/lib/python2.7/threading.py", line 855, in __bootstrap_inner pass File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 95, in run self._exc_info = sys.exc_info() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/app_ui.py", line 203, in _GetRootUiNode self._device.ReadFile(dtemp.name, force_pull=True)) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1161, in RunShellCommand output = handle_large_output(cmd, large_output) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1130, in handle_large_output return handle_large_command(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1112, in handle_large_command return handle_check_return(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1103, in handle_check_return return run(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1099, in run return self.adb.Shell(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 533, in Shell output = self._RunDeviceAdbCmd(args, timeout, retries, check_error=False) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 316, in _RunDeviceAdbCmd check_error=check_error) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 277, in _RunAdbCmd timeout, env=cls._ADB_ENV) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/cmd_helper.py", line 415, in GetCmdStatusAndOutputWithTimeout for data in _IterProcessStdout(process, timeout=timeout): File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/cmd_helper.py", line 277, in _IterProcessStdoutFcntl [child_fd], [], [], iter_aware_poll_interval) ******************************************************************************** Timed out. Dumping threads. ******************************************************************************** Stack dump for thread 'TimeoutThread-2-for-MainThread' ******************************************************************************** File: "/b/swarming/w/ir/.swarming_module/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File: "/b/swarming/w/ir/.swarming_module/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 93, in run self._ret = self._func(*self._args, **self._kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/app_ui.py", line 201, in _GetRootUiNode check_return=True) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1161, in RunShellCommand output = handle_large_output(cmd, large_output) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1130, in handle_large_output return handle_large_command(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1112, in handle_large_command return handle_check_return(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1103, in handle_check_return return run(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1099, in run return self.adb.Shell(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 533, in Shell output = self._RunDeviceAdbCmd(args, timeout, retries, check_error=False) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 316, in _RunDeviceAdbCmd check_error=check_error) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 277, in _RunAdbCmd timeout, env=cls._ADB_ENV) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/cmd_helper.py", line 415, in GetCmdStatusAndOutputWithTimeout for data in _IterProcessStdout(process, timeout=timeout): File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/cmd_helper.py", line 277, in _IterProcessStdoutFcntl [child_fd], [], [], iter_aware_poll_interval) ******************************************************************************** Timed out. Dumping threads. ******************************************************************************** Stack dump for thread 'TimeoutThread-3-for-MainThread' ******************************************************************************** File: "/b/swarming/w/ir/.swarming_module/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File: "/b/swarming/w/ir/.swarming_module/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 93, in run self._ret = self._func(*self._args, **self._kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/app_ui.py", line 201, in _GetRootUiNode check_return=True) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1161, in RunShellCommand output = handle_large_output(cmd, large_output) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1130, in handle_large_output return handle_large_command(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1112, in handle_large_command return handle_check_return(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1103, in handle_check_return return run(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1099, in run return self.adb.Shell(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 533, in Shell output = self._RunDeviceAdbCmd(args, timeout, retries, check_error=False) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 316, in _RunDeviceAdbCmd check_error=check_error) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 277, in _RunAdbCmd timeout, env=cls._ADB_ENV) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/cmd_helper.py", line 415, in GetCmdStatusAndOutputWithTimeout for data in _IterProcessStdout(process, timeout=timeout): File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/cmd_helper.py", line 277, in _IterProcessStdoutFcntl [child_fd], [], [], iter_aware_poll_interval) ******************************************************************************** Timed out. Dumping threads. ******************************************************************************** Stack dump for thread 'TimeoutThread-4-for-MainThread' ******************************************************************************** File: "/b/swarming/w/ir/.swarming_module/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner() File: "/b/swarming/w/ir/.swarming_module/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 93, in run self._ret = self._func(*self._args, **self._kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/app_ui.py", line 201, in _GetRootUiNode check_return=True) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1161, in RunShellCommand output = handle_large_output(cmd, large_output) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1130, in handle_large_output return handle_large_command(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1112, in handle_large_command return handle_check_return(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1103, in handle_check_return return run(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/device_utils.py", line 1099, in run return self.adb.Shell(cmd) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 533, in Shell output = self._RunDeviceAdbCmd(args, timeout, retries, check_error=False) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 316, in _RunDeviceAdbCmd check_error=check_error) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 277, in _RunAdbCmd timeout, env=cls._ADB_ENV) File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/cmd_helper.py", line 415, in GetCmdStatusAndOutputWithTimeout for data in _IterProcessStdout(process, timeout=timeout): File: "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/cmd_helper.py", line 277, in _IterProcessStdoutFcntl [child_fd], [], [], iter_aware_poll_interval) ******************************************************************************** Problem when trying to gather stack trace: Traceback (most recent call last): File "/b/swarming/w/ir/third_party/catapult/telemetry/telemetry/core/exceptions.py", line 72, in __init__ self._is_valid_dump, trace_output = app.GetStackTrace() File "/b/swarming/w/ir/third_party/catapult/telemetry/telemetry/internal/browser/browser.py", line 240, in GetStackTrace return self._browser_backend.GetStackTrace() File "/b/swarming/w/ir/third_party/catapult/common/py_trace_event/py_trace_event/trace_event_impl/decorators.py", line 52, in traced_function return func(*args, **kwargs) File "/b/swarming/w/ir/third_party/catapult/telemetry/telemetry/internal/backends/chrome/android_browser_backend.py", line 196, in GetStackTrace return self.platform_backend.GetStackTrace() File "/b/swarming/w/ir/third_party/catapult/telemetry/telemetry/internal/platform/android_platform_backend.py", line 640, in GetStackTrace ret = Decorate('UI dump', '\n'.join(self.GetSystemUi().ScreenDump())) File "/b/swarming/w/ir/third_party/catapult/devil/devil/android/app_ui.py", line 212, in ScreenDump return self._GetRootUiNode().Dump() File "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 198, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 163, in _JoinAll (len(alive_threads), len(self._threads))) CommandTimeoutError: Timed out waiting for 1 of 1 threads. Traceback (most recent call last): _RunGpuTest at content/test/gpu/gpu_tests/gpu_integration_test.py:155 self.RunActualGpuTest(url, *args) RunActualGpuTest at content/test/gpu/gpu_tests/webgl_conformance_integration_test.py:195 getattr(self, test_name)(test_path, *args[1:]) _RunConformanceTest at content/test/gpu/gpu_tests/webgl_conformance_integration_test.py:284 self._CheckTestCompletion() _CheckTestCompletion at content/test/gpu/gpu_tests/webgl_conformance_integration_test.py:274 'webglTestHarness._finished', timeout=self._GetTestTimeout()) traced_function at third_party/catapult/common/py_trace_event/py_trace_event/trace_event_impl/decorators.py:52 return func(*args, **kwargs) WaitForJavaScriptCondition at third_party/catapult/telemetry/telemetry/internal/actions/action_runner.py:263 return self._tab.WaitForJavaScriptCondition(*args, **kwargs) traced_function at third_party/catapult/common/py_trace_event/py_trace_event/trace_event_impl/decorators.py:52 return func(*args, **kwargs) WaitForJavaScriptCondition at third_party/catapult/telemetry/telemetry/internal/browser/web_contents.py:245 return self._inspector_backend.WaitForJavaScriptCondition(*args, **kwargs) traced_function at third_party/catapult/common/py_trace_event/py_trace_event/trace_event_impl/decorators.py:52 return func(*args, **kwargs) WaitForJavaScriptCondition at third_party/catapult/telemetry/telemetry/internal/backends/chrome_inspector/inspector_backend.py:307 self.GetCurrentConsoleOutputBuffer()) traced_function at third_party/catapult/common/py_trace_event/py_trace_event/trace_event_impl/decorators.py:52 return func(*args, **kwargs) Inner at third_party/catapult/telemetry/telemetry/internal/backends/chrome_inspector/inspector_backend.py:41 inspector_backend._ConvertExceptionFromInspectorWebsocket(e) traced_function at third_party/catapult/common/py_trace_event/py_trace_event/trace_event_impl/decorators.py:52 return func(*args, **kwargs) Inner at third_party/catapult/telemetry/telemetry/internal/backends/chrome_inspector/inspector_backend.py:38 return func(inspector_backend, *args, **kwargs) GetCurrentConsoleOutputBuffer at third_party/catapult/telemetry/telemetry/internal/backends/chrome_inspector/inspector_backend.py:196 return self._console.GetCurrentConsoleOutputBuffer(timeout) GetCurrentConsoleOutputBuffer at third_party/catapult/telemetry/telemetry/internal/backends/chrome_inspector/inspector_console.py:38 self._inspector_websocket.DispatchNotifications(timeout) DispatchNotifications at third_party/catapult/telemetry/telemetry/internal/backends/chrome_inspector/inspector_websocket.py:156 self._Receive(timeout) _Receive at third_party/catapult/telemetry/telemetry/internal/backends/chrome_inspector/inspector_websocket.py:181 raise WebSocketException(err) TimeoutException: Timeout after 300s while waiting for JavaScript:webglTestHarness._finished Exception thrown when trying to capture console output: TimeoutException('',) Locals: err : WebSocketTimeoutException('timed out',) timeout : 10 Restarting browser due to unexpected test failure
,
Dec 5
Happens on other devices as well. https://ci.chromium.org/p/chromium/builders/luci.chromium.ci/Android%20FYI%20Release%20%28Nexus%209%29/10985
,
Dec 5
jperaza@ is there a plan to fix that issue for all Chrome sub-processes? Is the work tracked under a bug? Should I block another bug on this one? Thanks.
,
Dec 6
Removing Infra>Client>Chrome. + mark -- it sounds like we believe that the issues might be related to turning on crashpad? Could you follow up on c#1?
,
Dec 7
gpu triage: Assigning to mark per #4
,
Dec 7
I think this will be fixed by: https://chromium-review.googlesource.com/c/chromium/src/+/1361597
,
Dec 10
The following revision refers to this bug: https://chromium.googlesource.com/chromium/src.git/+/d587561ae75deb5e1f310cca81e59ab5970e7e3f commit d587561ae75deb5e1f310cca81e59ab5970e7e3f Author: Joshua Peraza <jperaza@chromium.org> Date: Mon Dec 10 20:29:20 2018 android: Restore crash signal handlers for child processes The original tombstone producing signal handlers are still used by tests. This can be revisited later: https://bugs.chromium.org/p/chromium/issues/detail?id=912306 Bug: 911678, 912166, 912161 Change-Id: I55c0627bc431c6d61872a60d47423fd99567028f Reviewed-on: https://chromium-review.googlesource.com/c/1361597 Commit-Queue: Joshua Peraza <jperaza@chromium.org> Reviewed-by: Mark Mentovai <mark@chromium.org> Reviewed-by: Kenneth Russell <kbr@chromium.org> Cr-Commit-Position: refs/heads/master@{#615230} [modify] https://crrev.com/d587561ae75deb5e1f310cca81e59ab5970e7e3f/components/crash/content/app/crashpad_linux.cc
,
Dec 12
One other potential issue: Seeing timeouts inside Telemetry while attempting to gather a stack trace. Telemetry is trying to dump the UI and that operation is timing out. Our harness doesn't want to attempt to do this. Example failures: https://ci.chromium.org/p/chromium/builders/luci.chromium.try/android-marshmallow-arm64-rel/149070 https://chromium-swarm.appspot.com/task?id=41b7c000e7e83d10&refresh=10&show_raw=1 https://ci.chromium.org/p/chromium/builders/luci.chromium.try/android-marshmallow-arm64-rel/149062 https://chromium-swarm.appspot.com/task?id=41b7a71551a0a110&refresh=10&show_raw=1 Relevant stack trace from logs: Problem when trying to gather stack trace: Traceback (most recent call last): File "/b/swarming/w/ir/third_party/catapult/telemetry/telemetry/core/exceptions.py", line 72, in __init__ self._is_valid_dump, trace_output = app.GetStackTrace() File "/b/swarming/w/ir/third_party/catapult/telemetry/telemetry/internal/browser/browser.py", line 240, in GetStackTrace return self._browser_backend.GetStackTrace() File "/b/swarming/w/ir/third_party/catapult/common/py_trace_event/py_trace_event/trace_event_impl/decorators.py", line 52, in traced_function return func(*args, **kwargs) File "/b/swarming/w/ir/third_party/catapult/telemetry/telemetry/internal/backends/chrome/android_browser_backend.py", line 196, in GetStackTrace return self.platform_backend.GetStackTrace() File "/b/swarming/w/ir/third_party/catapult/telemetry/telemetry/internal/platform/android_platform_backend.py", line 640, in GetStackTrace ret = Decorate('UI dump', '\n'.join(self.GetSystemUi().ScreenDump())) File "/b/swarming/w/ir/third_party/catapult/devil/devil/android/app_ui.py", line 212, in ScreenDump return self._GetRootUiNode().Dump() File "/b/swarming/w/ir/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 198, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 163, in _JoinAll (len(alive_threads), len(self._threads))) CommandTimeoutError: Timed out waiting for 1 of 1 threads. We need a way to configure the call to self._GetRootUiNode().Dump() and not do that in our test harness.
,
Dec 12
Issue 914085 has been merged into this issue.
,
Dec 13
,
Jan 16
(6 days ago)
,
Jan 16
(6 days ago)
|
|||||||||||||||||||||
►
Sign in to add a comment |
|||||||||||||||||||||
Comment 1 by jperaza@google.com
, Dec 5