In appengine/api/urlfetch.py, several exceptions can be thrown if a url is unreachable, for example:

...
raise ConnectionClosedError('Connection closed unexpectedly by server at URL: ' + url)
raise DownloadError("Unable to fetch URL: " + url + error_detail)
raise DeadlineExceededError('Deadline exceeded while waiting for HTTP response from URL: ' + url)
raise DownloadError('Unable to connect to server at URL: ' + url)
...
among others.

Findit's http_retry_client only does a retry if the connection is successful, but a non-200 http status code is returned. However, Findit does not have exception handling from urlfetch, and thus the analysis dies.



Some examples of such failure occurring (from Findit's appengine logs):

Generator waterfall.process_flake_swarming_task_result_pipeline.ProcessFlakeSwarmingTaskResultPipeline(*(u'chromium.win', u'Win7 Tests (dbg)(1)', 54116, u'browser_tests on Windows-7-SP1', u'323e5fc679314110', 54249, u'OptionsWebUIExtendedTest.ShowPageWithHistory', 1), **{})#46140ea133e34bd2a11f29819519942e raised exception. ConnectionClosedError: Connection closed unexpectedly by server at URL: https://chromium-swarm.appspot.com/_ah/api/swarming/v1/task/323e5fc679314110/result
Traceback (most recent call last):
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:0d99c79.396648033247204792/third_party/pipeline_utils/appengine_third_party_pipeline_python_src_pipeline/pipeline.py", line 2158, in evaluate
    self, pipeline_key, root_pipeline_key, caller_output)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:0d99c79.396648033247204792/third_party/pipeline_utils/appengine_third_party_pipeline_python_src_pipeline/pipeline.py", line 1113, in _run_internal
    return self.run(*self.args, **self.kwargs)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:0d99c79.396648033247204792/waterfall/process_base_swarming_task_result_pipeline.py", line 86, in run
    task_id, self.HTTP_CLIENT)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:0d99c79.396648033247204792/waterfall/swarming_util.py", line 149, in GetSwarmingTaskResultById
    data = _SendRequestToServer(base_url, http_client)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:0d99c79.396648033247204792/waterfall/swarming_util.py", line 45, in _SendRequestToServer
    status_code, content = http_client.Get(url, headers=headers)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:0d99c79.396648033247204792/common/retry_http_client.py", line 110, in Get
    retry_backoff=retry_backoff, headers=headers)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:0d99c79.396648033247204792/common/retry_http_client.py", line 82, in _Retry
    status_code, content = self._Get(url, timeout_seconds, headers)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:0d99c79.396648033247204792/common/http_client_appengine.py", line 68, in _Get
    return self._SendRequest(url, urlfetch.GET, None, timeout, headers)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:0d99c79.396648033247204792/common/http_client_appengine.py", line 59, in _SendRequest
    follow_redirects=self.follow_redirects, validate_certificate=True)
  File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/urlfetch.py", line 271, in fetch
    return rpc.get_result()
  File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/apiproxy_stub_map.py", line 613, in get_result
    return self.__get_result_hook(self)
  File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/urlfetch.py", line 398, in _get_fetch_result
    'Connection closed unexpectedly by server at URL: ' + url)
ConnectionClosedError: Connection closed unexpectedly by server at URL: https://chromium-swarm.appspot.com/_ah/api/swarming/v1/task/323e5fc679314110/result


Generator waterfall.process_flake_swarming_task_result_pipeline.ProcessFlakeSwarmingTaskResultPipeline(*(u'chromium.win', u'Win7 Tests (dbg)(1)', 53966, u'browser_tests on Windows-7-SP1', u'32200baa1a891f10', 54074, u'MdSettingsUITest.ToggleJavaScript', 1), **{})#e9707ef7930c4b479e87d8f8277151ee raised exception. DownloadError: Unable to connect to server at URL: https://isolateserver.appspot.com/_ah/api/isolateservice/v1/retrieve
Traceback (most recent call last):
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/third_party/pipeline_utils/appengine_third_party_pipeline_python_src_pipeline/pipeline.py", line 2158, in evaluate
    self, pipeline_key, root_pipeline_key, caller_output)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/third_party/pipeline_utils/appengine_third_party_pipeline_python_src_pipeline/pipeline.py", line 1113, in _run_internal
    return self.run(*self.args, **self.kwargs)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/waterfall/process_base_swarming_task_result_pipeline.py", line 99, in run
    outputs_ref, self.HTTP_CLIENT)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/waterfall/swarming_util.py", line 158, in GetSwarmingTaskFailureLog
    return _DownloadTestResults(isolated_data, http_client)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/waterfall/swarming_util.py", line 306, in _DownloadTestResults
    data_for_output_json, http_client)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/waterfall/swarming_util.py", line 247, in _FetchOutputJsonInfoFromIsolatedServer
    content = _SendRequestToServer(url, http_client, post_data)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/waterfall/swarming_util.py", line 43, in _SendRequestToServer
    status_code, content = http_client.Post(url, post_data, headers=headers)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/common/retry_http_client.py", line 131, in Post
    retry_backoff=retry_backoff, headers=headers)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/common/retry_http_client.py", line 78, in _Retry
    status_code, content = self._Post(url, data, timeout_seconds, headers)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/common/http_client_appengine.py", line 71, in _Post
    return self._SendRequest(url, urlfetch.POST, data, timeout, headers)
  File "/base/data/home/apps/s~findit-for-me/waterfall-backend:808d293.396590759273648164/common/http_client_appengine.py", line 55, in _SendRequest
    follow_redirects=self.follow_redirects, validate_certificate=True)
  File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/urlfetch.py", line 271, in fetch
    return rpc.get_result()
  File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/apiproxy_stub_map.py", line 613, in get_result
    return self.__get_result_hook(self)
  File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/urlfetch.py", line 435, in _get_fetch_result
    raise DownloadError('Unable to connect to server at URL: ' + url)
DownloadError: Unable to connect to server at URL: https://isolateserver.appspot.com/_ah/api/isolateservice/v1/retrieve

swarming/isolated servers appear more susceptible to this error, though it can happen when trying to contact any url