shard scheduler crash |
||||
Issue description
EXCEPTION: Uncaught exception; terminating monitor_db
Traceback (most recent call last):
File "/usr/local/autotest/scheduler/monitor_db.py", line 186, in main_without_exception_handling
dispatcher.tick()
File "/usr/local/autotest/site-packages/statsd/timer.py", line 95, in _decorator
return function(*args, **kwargs)
File "/usr/local/autotest/scheduler/site_monitor_db.py", line 106, in tick
super(SiteDispatcher, self).tick()
File "/usr/local/autotest/scheduler/monitor_db.py", line 359, in tick
self._schedule_special_tasks()
File "/usr/local/autotest/site-packages/statsd/timer.py", line 95, in _decorator
return function(*args, **kwargs)
File "/usr/local/autotest/scheduler/site_monitor_db.py", line 151, in _schedule_special_tasks
super(SiteDispatcher, self)._schedule_special_tasks()
File "/usr/local/autotest/scheduler/monitor_db.py", line 711, in _schedule_special_tasks
self.add_agent_task(self._get_agent_task_for_special_task(task))
File "/usr/local/autotest/scheduler/monitor_db.py", line 619, in _get_agent_task_for_special_task
return agent_task_class(task=special_task)
File "/usr/local/autotest/scheduler/prejob_task.py", line 365, in __init__
self._set_milestone(labels)
File "/usr/local/autotest/scheduler/prejob_task.py", line 377, in _set_milestone
cros_image = _CrosImage(_get_cros_version(label))
File "/usr/local/autotest/scheduler/prejob_task.py", line 336, in __init__
self.group = match.group('group')
AttributeError: 'NoneType' object has no attribute 'group'
,
Oct 20 2016
,
Oct 20 2016
,
Oct 20 2016
The issue is fixed in the revert: https://chromium-review.googlesource.com/#/c/401020/ Lab was updated and open again.
,
Oct 20 2016
,
Oct 21 2016
The following revision refers to this bug: https://chromium.googlesource.com/chromiumos/third_party/autotest/+/02d7e74c7670ad8dc63b6ea08667522adb229f5a commit 02d7e74c7670ad8dc63b6ea08667522adb229f5a Author: Allen Li <ayatane@chromium.org> Date: Fri Oct 14 22:30:36 2016 [autotest] Add milestone to special_task metrics v2 BUG=chromium:650481, chromium:657664 TEST=Run suite in local autotest Change-Id: I32a064c43725930e7cca9bca864eae8172d45f66 Reviewed-on: https://chromium-review.googlesource.com/400940 Commit-Ready: Allen Li <ayatane@chromium.org> Tested-by: Allen Li <ayatane@chromium.org> Reviewed-by: Aviv Keshet <akeshet@chromium.org> [modify] https://crrev.com/02d7e74c7670ad8dc63b6ea08667522adb229f5a/server/cros/provision.py [modify] https://crrev.com/02d7e74c7670ad8dc63b6ea08667522adb229f5a/scheduler/agent_task.py [modify] https://crrev.com/02d7e74c7670ad8dc63b6ea08667522adb229f5a/scheduler/prejob_task.py |
||||
►
Sign in to add a comment |
||||
Comment 1 by shuqianz@chromium.org
, Oct 20 2016I notice the error in the push shard scheduler log: 10/19 17:15:17.794 DEBUG| monitor_db:0332| Calling _schedule_delay_tasks(). 10/19 17:15:17.795 INFO | base:0063| POST http://172.25.66.62:9800/_bulk [status:200 request:0.047s] 10/19 17:15:17.796 DEBUG| base:0065| > {"index": {"_type": "host_history", "_index": "localhost"}} {"status": "Ready", "task_id": 6, "hostname": "chromeos4-row10-rack9-host15", "board": "quawks", "task_name": "Repair", "pools": ["bvt"], "dbg_str": "Task: Special Task 6 (host chromeos4-row10-rack9-host15 , task Repair, time 2016-10-19 17:01:30)", "time_recorded": 1476922156.314454} {"index": {"_type": "host_history", "_index": "localhost"}} {"status": "Verifying", "task_id": 1, "hostname": "chromeos4-row10-rack9-host11", "board": "quawks", "task_name": "Verify", "pools": ["bvt"], "dbg_str": "Task: Special Task 1 (host chromeos4-row10-rack9-ho st11, task Verify, time 2016-10-19 16:51:32)", "time_recorded": 1476921093.670413} {"index": {"_type": "host_history", "_index": "localhost"}} {"status": "Ready", "task_id": 8, "hostname": "chromeos4-row10-rack9-host11", "board": "quawks", "task_name": "Repair", "pools": ["bvt"], "dbg_str": "Task: Special Task 8 (host chromeos4-row10-rack9-host11 , task Repair, time 2016-10-19 17:04:33)", "time_recorded": 1476922352.550368} {"index": {"_type": "host_history", "_index": "localhost"}} {"status": "Pending", "job_id": 196, "hostname": "chromeos4-row10-rack9-host15", "owner": "chromeos-test", "parent_job_id": null, "board": "quawks", "pools": ["bvt"], "dbg_str": "HQE: 196, for job: 196", " job_name": "quawks-release/R54-8743.65.0/paygen_au_beta/autoupdate_EndToEndTest_paygen_au_beta_full_8743.65.0", "time_recorded": 1476922175.182609} {"index": {"_type": "host_history", "_index": "localhost"}} {"status": "Running", "job_id": 196, "hostname": "chromeos4-row10-rack9-host15", "owner": "chromeos-test", "parent_job_id": null, "board": "quawks", "pools": ["bvt"], "dbg_str": "HQE: 196, for job: 196", "job_name": "quawks-release/R54-8743.65.0/paygen_au_beta/autoupdate_EndToEndTest_paygen_au_beta_full_8743.65.0", "time_recorded": 1476922176.334523} {"index": {"_type": "host_history", "_index": "localhost"}} {"status": "Ready", "task_id": 1, "hostname": "chromeos4-row10-rack9-host11", "board": "quawks", "task_name": "Verify", "pools": ["bvt"], "dbg_str": "Task: Special Task 1 (host chromeos4-row10-rack9-host11, task Verify, time 2016-10-19 16:51:32)", "time_recorded": 1476921125.678345} {"index": {"_type": "host_history", "_index": "localhost"}} {"status": "Verifying", "task_id": 2, "hostname": "chromeos4-row10-rack9-host15", "board": "quawks", "task_name": "Verify", "pools": ["bvt"], "dbg_str": "Task: Special Task 2 (host chromeos4-row10-rack9-host15, task Verify, time 2016-10-19 16:52:07)", "time_recorded": 1476921129.134566} {"index": {"_type": "host_history", "_index": "localhost"}} {"status": "Ready", "task_id": 2, "hostname": "chromeos4-row10-rack9-host15", "board": "quawks", "task_name": "Verify", "pools": ["bvt"], "dbg_str": "Task: Special Task 2 (host chromeos4-row10-rack9-host15, task Verify, time 2016-10-19 16:52:07)", "time_recorded": 1476921161.298694} {"index": {"_type": "host_history", "_index": "localhost"}} {"status": "Pending", "job_id": 193, "hostname": "chromeos4-row10-rack9-host15", "owner": "autotest_system", "parent_job_id": null, "board": "quawks", "pools": ["bvt"], "dbg_str": "HQE: 193, for job: 193", "job_name": "powerwash", "time_recorded": 1476921506.210526} {"index": {"_type": "host_history", "_index": "localhost"}} {"status": "Running", "job_id": 193, "hostname": "chromeos4-row10-rack9-host15", "owner": "autotest_system", "parent_job_id": null, "board": "quawks", "pools": ["bvt"], "dbg_str": "HQE: 193, for job: 193", "job_name": "powerwash", "time_recorded": 1476921507.274243} 10/19 17:15:17.796 DEBUG| base:0066| < {"took":1,"errors":true,"items":[{"index":{"_index":"localhost","_type":"host_history","_id":null,"status":404,"error":"IndexMissingException[[localhost] missing]"}},{"index":{"_index":"localhost","_type":"host_history","_id":null,"status":404,"error":"IndexMissingException[[localhost] missing]"}},{"index":{"_index":"localhost","_type":"host_history","_id":null,"status":404,"error":"IndexMissingException[[localhost] missing]"}},{"index":{"_index":"localhost","_type":"host_history","_id":null,"status":404,"error":"IndexMissingException[[localhost] missing]"}},{"index":{"_index":"localhost","_type":"host_history","_id":null,"status":404,"error":"IndexMissingException[[localhost] missing]"}},{"index":{"_index":"localhost","_type":"host_history","_id":null,"status":404,"error":"IndexMissingException[[localhost] missing]"}},{"index":{"_index":"localhost","_type":"host_history","_id":null,"status":404,"error":"IndexMissingException[[localhost] missing]"}},{"index":{"_index":"localhost","_type":"host_history","_id":null,"status":404,"error":"IndexMissingException[[localhost] missing]"}},{"index":{"_index":"localhost","_type":"host_history","_id":null,"status":404,"error":"IndexMissingException[[localhost] missing]"}},{"index":{"_index":"localhost","_type":"host_history","_id":null,"status":404,"error":"IndexMissingException[[localhost] missing]"}}]} 10/19 17:15:17.798 DEBUG| monitor_db:0332| Calling _schedule_running_host_queue_entries(). 10/19 17:15:17.799 ERROR| es_utils:0228| ('10 document(s) failed to index.', [{u'index': {u'status': 404, u'_type': u'host_history', u'_id': None, u'error': u'IndexMissingException[[localhost] missing]', u'_index': u'localhost'}}, {u'index': {u'status': 404, u'_type': u'host_history', u'_id': None, u'error': u'IndexMissingException[[localhost] missing]', u'_index': u'localhost'}}, {u'index': {u'status': 404, u'_type': u'host_history', u'_id': None, u'error': u'IndexMissingException[[localhost] missing]', u'_index': u'localhost'}}, {u'index': {u'status': 404, u'_type': u'host_history', u'_id': None, u'error': u'IndexMissingException[[localhost] missing]', u'_index': u'localhost'}}, {u'index': {u'status': 404, u'_type': u'host_history', u'_id': None, u'error': u'IndexMissingException[[localhost] missing]', u'_index': u'localhost'}}, {u'index': {u'status': 404, u'_type': u'host_history', u'_id': None, u'error': u'IndexMissingException[[localhost] missing]', u'_index': u'localhost'}}, {u'index': {u'status': 404, u'_type': u'host_history', u'_id': None, u'error': u'IndexMissingException[[localhost] missing]', u'_index': u'localhost'}}, {u'index': {u'status': 404, u'_type': u'host_history', u'_id': None, u'error': u'IndexMissingException[[localhost] missing]', u'_index': u'localhost'}}, {u'index': {u'status': 404, u'_type': u'host_history', u'_id': None, u'error': u'IndexMissingException[[localhost] missing]', u'_index': u'localhost'}}, {u'index': {u'status': 404, u'_type': u'host_history', u'_id': None, u'error': u'IndexMissingException[[localhost] missing]', u'_index': u'localhost'}}]) 10/19 17:15:17.800 DEBUG| retry:0155| Converted retries value: False -> Retry(total=False, connect=None, read=None, redirect=0) 10/19 17:15:17.814 DEBUG| monitor_db:0332| Calling _schedule_special_tasks(). 10/19 17:15:17.816 DEBUG| monitor_db:0332| Calling _schedule_new_jobs(). :