HWTest timeout for sentry-release |
||||
Issue descriptionhttps://uberchromegw.corp.google.com/i/chromeos/builders/sentry-release/builds/574 ************************************************************ ** Finished Stage HWTest [bvt-inline] - Tue, 15 Nov 2016 08:13:07 -0800 (PST) ************************************************************ Traceback (most recent call last): File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 441, in _Run self._task(*self._task_args, **self._task_kwargs) File "/b/cbuild/internal_master/chromite/cbuildbot/stages/generic_stages.py", line 560, in Run self.PerformStage() File "/b/cbuild/internal_master/chromite/cbuildbot/stages/test_stages.py", line 385, in PerformStage debug=debug, subsystems=subsystems, skip_duts_check=skip_duts_check) File "/b/cbuild/internal_master/chromite/lib/failures_lib.py", line 172, in wrapped_functor return functor(*args, **kwargs) File "/b/cbuild/internal_master/chromite/cbuildbot/commands.py", line 948, in RunHWTestSuite _HWTestWait(cmd, job_id, **swarming_args) File "/b/cbuild/internal_master/chromite/cbuildbot/commands.py", line 1187, in _HWTestWait **kwargs) File "/b/cbuild/internal_master/chromite/cbuildbot/swarming_lib.py", line 153, in RunSwarmingCommandWithRetries return retry_util.RetryCommand(RunSwarmingCommand, max_retry, *args, **kwargs) File "/b/cbuild/internal_master/chromite/lib/retry_util.py", line 218, in RetryCommand return GenericRetry(ShouldRetry, max_retry, functor, *args, **kwargs) File "/b/cbuild/internal_master/chromite/lib/retry_util.py", line 115, in GenericRetry ret = functor(*args, **kwargs) File "/b/cbuild/internal_master/chromite/cbuildbot/swarming_lib.py", line 92, in RunSwarmingCommand result = cros_build_lib.RunCommand(swarming_cmd, *args, **kwargs) File "/b/cbuild/internal_master/chromite/lib/cros_build_lib.py", line 593, in RunCommand (cmd_result.output, cmd_result.error) = proc.communicate(input) File "/usr/lib/python2.7/subprocess.py", line 796, in communicate self.wait() File "/usr/lib/python2.7/subprocess.py", line 1376, in wait pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) File "/usr/lib/python2.7/subprocess.py", line 476, in _eintr_retry_call return func(*args) File "/b/cbuild/internal_master/chromite/lib/cros_build_lib.py", line 340, in _KillChildProcess if not signals.RelaySignal(original_handler, signum, frame): File "/b/cbuild/internal_master/chromite/lib/signals.py", line 26, in RelaySignal handler(signum, frame) File "/b/cbuild/internal_master/chromite/lib/commandline.py", line 731, in _DefaultHandler signum, 'Received signal %i; shutting down' % (signum,)) _ShutDownException: (15, 'Received signal 15; shutting down') [1;33m08:13:07: WARNING: No output from <_BackgroundTask(_BackgroundTask-7:7:4, started)> for 8640 seconds[0m File "chromite/bin/cbuildbot", line 168, in <module> DoMain() File "chromite/bin/cbuildbot", line 164, in DoMain commandline.ScriptWrapperMain(FindTarget) File "/b/cbuild/internal_master/chromite/lib/commandline.py", line 834, in ScriptWrapperMain ret = target(argv[1:]) File "/b/cbuild/internal_master/chromite/scripts/cbuildbot.py", line 1307, in main _RunBuildStagesWrapper(options, site_config, build_config) File "/b/cbuild/internal_master/chromite/scripts/cbuildbot.py", line 248, in _RunBuildStagesWrapper if not builder.Run(): File "/b/cbuild/internal_master/chromite/cbuildbot/builders/generic_builders.py", line 316, in Run self.RunStages() File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 474, in RunStages super(DistributedBuilder, self).RunStages() File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 334, in RunStages self._RunDefaultTypeBuild() File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 321, in _RunDefaultTypeBuild self.RunBuildStages() File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 282, in RunBuildStages with parallel.BackgroundTaskRunner(task_runner) as queue: File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__ return self.gen.next() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 745, in BackgroundTaskRunner with _BackgroundTask.ParallelTasks(steps, halt_on_error=halt_on_error): File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__ return self.gen.next() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 532, in ParallelTasks task.start() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 388, in start return multiprocessing.Process.start(self) File "/usr/lib/python2.7/multiprocessing/process.py", line 130, in start self._popen = Popen(self) File "/usr/lib/python2.7/multiprocessing/forking.py", line 126, in __init__ code = process_obj._bootstrap() File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 399, in run errors = self._Run() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 441, in _Run self._task(*self._task_args, **self._task_kwargs) File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 602, in TaskRunner task(*x, **task_kwargs) File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 128, in _RunBackgroundStagesForBoardAndMarkAsSuccessful self._RunBackgroundStagesForBoard(builder_run, board) File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 220, in _RunBackgroundStagesForBoard lambda: self._RunHWTests(builder_run, board), File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 674, in RunParallelSteps halt_on_error=halt_on_error): File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__ return self.gen.next() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 532, in ParallelTasks task.start() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 388, in start return multiprocessing.Process.start(self) File "/usr/lib/python2.7/multiprocessing/process.py", line 130, in start self._popen = Popen(self) File "/usr/lib/python2.7/multiprocessing/forking.py", line 126, in __init__ code = process_obj._bootstrap() File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 399, in run errors = self._Run() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 441, in _Run self._task(*self._task_args, **self._task_kwargs) File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 220, in <lambda> lambda: self._RunHWTests(builder_run, board), File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 112, in _RunHWTests self._RunParallelStages(parallel_stages) File "/b/cbuild/internal_master/chromite/cbuildbot/builders/generic_builders.py", line 118, in _RunParallelStages parallel.RunParallelSteps(steps) File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 675, in RunParallelSteps pass File "/usr/lib/python2.7/contextlib.py", line 24, in __exit__ self.gen.next() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 546, in ParallelTasks task_errors = task.Wait() File "/b/cbuild/internal_master/chromite/lib/parallel.py", line 363, in Wait traceback.print_stack()
,
Nov 16 2016
see this kind of trace quite a bit, but not clear what the root cause.
,
Nov 16 2016
It looks like a for the HWTests but I can't say for sure.
,
Nov 16 2016
,
Nov 21 2016
Probably a duplicate of Issue 666070: https://uberchromegw.corp.google.com/i/chromeos/builders/sentry-release/builds/574 http://cautotest.corp.google.com/afe/#tab_id=view_job&object_id=85760830 https://pantheon.corp.google.com/storage/browser/chromeos-autotest-results/85760830-chromeos-test/hostless/debug 11/15 08:49:15.759 ERROR| db:0169| 08:49:15 11/15/16: An operational error occured during a database operation: (2006, 'MySQL server has gone away') |
||||
►
Sign in to add a comment |
||||
Comment 1 by skau@chromium.org
, Nov 15 2016Components: Infra>Client>ChromeOS