Webstore is experiencing sync deadline exceeded failures, gang could you please take a look?
sglobal:rpc_server_time_in_usec:average.cws-afapi.afapiserver{rpc_command=ListInstalledItems,rpc_service_name=ConsumerService,service=chrome-webstore,user=chrome-webstore}: 2764774.0808731
0620 09:14:43.295000 12 EM-Thread-Shared EventManager in BackendModuleBuilder-0 AggregateFuture$RunningState.handleException Got more than one input Future failure. Logging failures after the first
com.google.net.rpc3.client.RpcClientException: <eye3-stackless title='/ChromeSyncDataService.ListWhitelists, DEADLINE_EXCEEDED'/>DEADLINE_EXCEEDED;personalization.chrome.sync.dataservice.proto/ChromeSyncDataService.ListWhitelists;Deadline=5.0s;StartTimeMs=1466439278294;tcp;Deadline(sec)=5.0;ResFormat=UNCOMPRESSED;Originator=gfe-prod;Tag=;NoServerTime;LogBytes=256;Non-FailFast;EffSecLevel=none;ReqFormat=UNCOMPRESSED;ReqID=fe507dce14ed1ebb;GlobalID=46346d807f4b0500;Server=10.54.90.65:4455
at com.google.net.rpc3.client.util.RpcAbstractCheckedFuture.defaultMapResult(RpcAbstractCheckedFuture.java:316)
at com.google.net.rpc3.client.util.RpcCheckedFuture.mapResult(RpcCheckedFuture.java:42)
at com.google.net.rpc3.client.util.RpcCheckedFuture.mapResult(RpcCheckedFuture.java:19)
at com.google.net.rpc3.client.util.RpcAbstractCheckedFuture.internalGet(RpcAbstractCheckedFuture.java:202)
at com.google.net.rpc3.client.util.RpcAbstractCheckedFuture.get(RpcAbstractCheckedFuture.java:243)
at com.google.common.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:196)
at com.google.common.util.concurrent.Futures.getDone(Futures.java:1336)
at com.google.common.util.concurrent.AbstractTransformFuture.run(AbstractTransformFuture.java:103)
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:441)
at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:143)
at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:132)
at com.google.net.rpc3.client.util.RpcAbstractCheckedFuture.complete(RpcAbstractCheckedFuture.java:300)
at com.google.net.rpc3.client.util.RpcAbstractCheckedFuture.failure(RpcAbstractCheckedFuture.java:275)
at com.google.net.rpc3.client.RpcStub$RpcCallbackDispatcher$1.runInContext(RpcStub.java:857)
at com.google.tracing.TraceContext$TraceContextRunnable$1.run(TraceContext.java:449)
at com.google.tracing.CurrentContext.runInContext(CurrentContext.java:256)
at com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContextNoUnref(TraceContext.java:313)
at com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContext(TraceContext.java:305)
at com.google.tracing.TraceContext$TraceContextRunnable.run(TraceContext.java:446)
at com.google.net.rpc3.client.RpcStub$RpcCallbackDispatcher.runCallback(RpcStub.java:893)
at com.google.net.rpc3.client.RpcStub$RpcCallbackDispatcher.rpcFinished(RpcStub.java:903)
at com.google.net.rpc3.client.RpcStub$RpcCallbackDispatcher.failure(RpcStub.java:888)
at com.google.net.rpc3.impl.client.RpcClientInternalContext.runCallbacks(RpcClientInternalContext.java:1237)
at com.google.net.rpc3.impl.client.RpcClientInternalContext.finishRpcAndNotifyApp(RpcClientInternalContext.java:1134)
at com.google.net.rpc3.client.loadbalancer.ReplicatedRpcChannel$InterceptedCallback.finishRpc(ReplicatedRpcChannel.java:1050)
at com.google.net.rpc3.client.loadbalancer.ReplicatedRpcChannel$InterceptedCallback.rpcFinished(ReplicatedRpcChannel.java:912)
at com.google.net.rpc3.client.loadbalancer.ReplicatedRpcChannel$InterceptedCallback.failure(ReplicatedRpcChannel.java:879)
at com.google.net.rpc3.impl.client.RpcClientInternalContext.runCallbacks(RpcClientInternalContext.java:1237)
at com.google.net.rpc3.impl.client.RpcClientInternalContext.access$200(RpcClientInternalContext.java:119)
at com.google.net.rpc3.impl.client.RpcClientInternalContext$1.run(RpcClientInternalContext.java:1130)
at com.google.net.eventmanager.AbstractFutureTask$Sync.innerRun(AbstractFutureTask.java:260)
at com.google.net.eventmanager.AbstractFutureTask.run(AbstractFutureTask.java:121)
at com.google.net.eventmanager.EventManagerImpl.runTask(EventManagerImpl.java:594)
at com.google.net.eventmanager.EventManagerImpl.internalRunWorkerLoop(EventManagerImpl.java:1012)
at com.google.net.eventmanager.EventManagerImpl.runWorkerLoop(EventManagerImpl.java:892)
at com.google.net.eventmanager.WorkerThreadInfo.runWorkerLoop(WorkerThreadInfo.java:161)
at com.google.net.eventmanager.EventManagerImpl$WorkerThread.run(EventManagerImpl.java:1877)
Comment 1 by plaree@chromium.org
, Jun 20 2016