We saw the same error on several builders, which all displayed it incorrectly.

https://uberchromegw.corp.google.com/i/chromiumos.tryserver/builders/gcc_toolchain/builds/163
https://uberchromegw.corp.google.com/i/chromiumos.tryserver/builders/llvm_next_toolchain/builds/53
https://uberchromegw.corp.google.com/i/chromiumos.tryserver/builders/llvm_next_toolchain/builds/52

It appears that the failure was raised in the stages's _Finish() method, but that GenericStage.Run() and/or ._RunStage() didn't catch and handle it appropriately. This meant that no stage was blamed for the failure, and our waterfall incorrectly displayed success for whatever stage was at fault.

Very confusing.


Exception thrown, but all stages marked successful. This is an internal error,
because the stage that threw the exception should be marked as failing.
cbuildbot: Unhandled exception:
Traceback (most recent call last):
  File "chromite/bin/cbuildbot", line 168, in <module>
    DoMain()
  File "chromite/bin/cbuildbot", line 164, in DoMain
    commandline.ScriptWrapperMain(FindTarget)
  File "/b/cbuild/internal_master/chromite/lib/commandline.py", line 834, in ScriptWrapperMain
    ret = target(argv[1:])
  File "/b/cbuild/internal_master/chromite/scripts/cbuildbot.py", line 1317, in main
    logging.info('One stage exited early: %s', ex)
  File "/b/cbuild/internal_master/chromite/scripts/cbuildbot.py", line 1314, in main
    _RunBuildStagesWrapper(options, site_config, build_config)
  File "/b/cbuild/internal_master/chromite/scripts/cbuildbot.py", line 248, in _RunBuildStagesWrapper
    if not builder.Run():
  File "/b/cbuild/internal_master/chromite/cbuildbot/builders/generic_builders.py", line 316, in Run
    self.RunStages()
  File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 345, in RunStages
    self._RunDefaultTypeBuild()
  File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 330, in _RunDefaultTypeBuild
    self.RunEarlySyncAndSetupStages()
  File "/b/cbuild/internal_master/chromite/cbuildbot/builders/simple_builders.py", line 264, in RunEarlySyncAndSetupStages
    self._RunStage(android_stages.AndroidMetadataStage)
  File "/b/cbuild/internal_master/chromite/cbuildbot/builders/generic_builders.py", line 107, in _RunStage
    return stage_instance.Run()
  File "/b/cbuild/internal_master/chromite/cbuildbot/stages/generic_stages.py", line 648, in Run
    self._Finish()
  File "/b/cbuild/internal_master/chromite/cbuildbot/stages/android_stages.py", line 144, in _Finish
    self._WriteAndroidVersionToMetadata()
  File "/b/cbuild/internal_master/chromite/cbuildbot/stages/android_stages.py", line 137, in _WriteAndroidVersionToMetadata
    self.UploadMetadata(filename=constants.PARTIAL_METADATA_JSON)
  File "/b/cbuild/internal_master/chromite/lib/failures_lib.py", line 172, in wrapped_functor
    return functor(*args, **kwargs)
  File "/b/cbuild/internal_master/chromite/cbuildbot/stages/generic_stages.py", line 1056, in UploadMetadata
    self.UploadArtifact(filename, archive=False)
  File "/b/cbuild/internal_master/chromite/lib/failures_lib.py", line 172, in wrapped_functor
    return functor(*args, **kwargs)
  File "/b/cbuild/internal_master/chromite/cbuildbot/stages/generic_stages.py", line 992, in UploadArtifact
    update_list=True, acl=self.acl)
  File "/b/cbuild/internal_master/chromite/lib/failures_lib.py", line 172, in wrapped_functor
    return functor(*args, **kwargs)
  File "/b/cbuild/internal_master/chromite/cbuildbot/commands.py", line 1843, in UploadArchivedFile
    _UploadPathToGS(file_path, upload_urls, debug, timeout, acl=acl)
  File "/b/cbuild/internal_master/chromite/lib/failures_lib.py", line 187, in wrapped_functor
    raise self.category_exception(exc_infos=exc_infos)
chromite.lib.failures_lib.GSUploadFailure: <class 'chromite.lib.gs.GSCommandError'>: return code: 1; command: /b/cbuild/internal_master/.cache/common/gsutil_4.19.tar.gz/gsutil/gsutil -o 'Boto:num_retries=10' -m cp -v -- /b/cbuild/internal_master/trybot_archive/trybot-squawks-llvm-next-toolchain/R57-9170.0.0-b52/partial-metadata.json gs://chromeos-image-archive/trybot-squawks-llvm-next-toolchain/R57-9170.0.0-b52/partial-metadata.json
Copying file:///b/cbuild/internal_master/trybot_archive/trybot-squawks-llvm-next-toolchain/R57-9170.0.0-b52/partial-metadata.json [Content-Type=text/plain]...
Uploading   ...toolchain/R57-9170.0.0-b52/partial-metadata.json: 0 B/1.79 KiB    
Uploading   ...toolchain/R57-9170.0.0-b52/partial-metadata.json: 1.79 KiB/1.79 KiB    
ServiceException: 503 Backend Error
CommandException: 1 file/object could not be transferred.

cmd=['/b/cbuild/internal_master/.cache/common/gsutil_4.19.tar.gz/gsutil/gsutil', '-o', 'Boto:num_retries=10', '-m', 'cp', '-v', '--', '/b/cbuild/internal_master/trybot_archive/trybot-squawks-llvm-next-toolchain/R57-9170.0.0-b52/partial-metadata.json', 'gs://chromeos-image-archive/trybot-squawks-llvm-next-toolchain/R57-9170.0.0-b52/partial-metadata.json'], extra env={'BOTO_CONFIG': '/b/build/site_config/.boto'}
Traceback (most recent call last):
  File "/b/cbuild/internal_master/chromite/lib/failures_lib.py", line 172, in wrapped_functor
    return functor(*args, **kwargs)
  File "/b/cbuild/internal_master/chromite/cbuildbot/commands.py", line 1824, in _UploadPathToGS
    recursive=True)
  File "/b/cbuild/internal_master/chromite/lib/gs.py", line 584, in CopyInto
    **kwargs)
  File "/b/cbuild/internal_master/chromite/lib/gs.py", line 874, in Copy
    raise
  File "/b/cbuild/internal_master/chromite/lib/gs.py", line 851, in Copy
    result = self.DoCommand(cmd, **kwargs)
  File "/b/cbuild/internal_master/chromite/lib/gs.py", line 782, in DoCommand
    raise GSCommandError(e.msg, e.result, e.exception)
GSCommandError: return code: 1; command: /b/cbuild/internal_master/.cache/common/gsutil_4.19.tar.gz/gsutil/gsutil -o 'Boto:num_retries=10' -m cp -v -- /b/cbuild/internal_master/trybot_archive/trybot-squawks-llvm-next-toolchain/R57-9170.0.0-b52/partial-metadata.json gs://chromeos-image-archive/trybot-squawks-llvm-next-toolchain/R57-9170.0.0-b52/partial-metadata.json
Copying file:///b/cbuild/internal_master/trybot_archive/trybot-squawks-llvm-next-toolchain/R57-9170.0.0-b52/partial-metadata.json [Content-Type=text/plain]...
Uploading   ...toolchain/R57-9170.0.0-b52/partial-metadata.json: 0 B/1.79 KiB    
Uploading   ...toolchain/R57-9170.0.0-b52/partial-metadata.json: 1.79 KiB/1.79 KiB    
ServiceException: 503 Backend Error
CommandException: 1 file/object could not be transferred.