Excessive Goma failures in LUCI SwarmBucket builds. |
||
Issue descriptionLooking at the cause of failure: GOMA:clang++ compiling ../../third_party/WebKit/Source/core/css/PropertyRegistry.cpp:*ERROR*: compiler_proxy:2290ms: reached max number of active fail fallbacks Example: - LUCI build: https://luci-milo.appspot.com/swarming/task/370c0c05a6b32b10 - Corresponding BuildBot build: https://luci-milo.appspot.com/buildbot/tryserver.chromium.mac/mac_chromium_rel_ng/489140 [2086/4430] CXX obj/third_party/WebKit/Source/core/css/css_1/PropertyRegistry.o FAILED: obj/third_party/WebKit/Source/core/css/css_1/PropertyRegistry.o export DEVELOPER_DIR=/b/s/w/ir/cache/builder/mac/src/build/mac_files/Xcode.app; /b/s/w/ir/cache/goma_client/gomacc ../../third_party/llvm-build/Release+Asserts/bin/clang++ -MMD -MF obj/third_party/WebKit/Source/core/css/css_1/PropertyRegistry.o.d -DV8_DEPRECATION_WARNINGS -DDCHECK_ALWAYS_ON=1 -DNO_TCMALLOC -DFULL_SAFE_BROWSING -DSAFE_BROWSING_CSD -DSAFE_BROWSING_DB_LOCAL -DCHROMIUM_BUILD -DFIELDTRIAL_TESTING_ENABLED -DCR_CLANG_REVISION=\"305735-2\" -DCR_XCODE_VERSION=0832 -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D_FORTIFY_SOURCE=2 -D__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORE=0 -DNDEBUG -DNVALGRIND -DDYNAMIC_ANNOTATIONS_ENABLED=0 -DBLINK_CORE_IMPLEMENTATION=1 -DANGLE_ENABLE_RELEASE_ASSERTS -DBLINK_IMPLEMENTATION=1 -DINSIDE_BLINK -DU_USING_ICU_NAMESPACE=0 -DU_ENABLE_DYLOAD=0 -DU_STATIC_IMPLEMENTATION -DICU_UTIL_DATA_IMPL=ICU_UTIL_DATA_FILE -DUCHAR_TYPE=uint16_t -DENABLE_IPC_FUZZER -DSK_IGNORE_LINEONLY_AA_CONVEX_PATH_OPTS -DSK_HAS_PNG_LIBRARY -DSK_HAS_WEBP_LIBRARY -DSK_HAS_JPEG_LIBRARY -DSK_SUPPORT_GPU=1 -DSK_BUILD_FOR_MAC -DGOOGLE_PROTOBUF_NO_RTTI -DGOOGLE_PROTOBUF_NO_STATIC_INITIALIZER -DHAVE_PTHREAD -DUSE_LIBJPEG_TURBO=1 -DLIBXSLT_STATIC -I../.. -Igen -I../../third_party/WebKit/Source -I../../third_party/WebKit -Igen/blink -Igen/third_party/WebKit -I../../third_party/khronos -I../../gpu -I../../third_party/libwebp/src -I../../third_party/WebKit/Source -Igen/blink -I../../third_party/ced/src -I../../third_party/icu/source/common -I../../third_party/icu/source/i18n -I../../skia/config -I../../skia/ext -I../../third_party/skia/include/c -I../../third_party/skia/include/config -I../../third_party/skia/include/core -I../../third_party/skia/include/effects -I../../third_party/skia/include/encode -I../../third_party/skia/include/images -I../../third_party/skia/include/lazy -I../../third_party/skia/include/pathops -I../../third_party/skia/include/pdf -I../../third_party/skia/include/pipe -I../../third_party/skia/include/ports -I../../third_party/skia/include/utils -I../../third_party/skia/third_party/vulkan -I../../third_party/skia/include/gpu -I../../third_party/skia/src/gpu -I../../third_party/skia/src/sksl -I../../third_party/libwebm/source -I../../third_party/protobuf/src -Igen/protoc_out -I../../third_party/protobuf/src -I../../third_party/boringssl/src/include -I../../third_party/libjpeg_turbo -I../../third_party/iccjpeg -I../../third_party/libpng -I../../third_party/zlib -I../../third_party/ots/include -I../../v8/include -Igen/v8/include -I../../third_party/angle/include -I../../third_party/angle/src/common/third_party/base -Igen/angle -I../../third_party/libxml/src/include -I../../third_party/libxml/mac/include -I../../third_party/libxslt/src -I../../third_party/snappy/src -I../../third_party/snappy/mac -fno-strict-aliasing -fstack-protector -Wno-builtin-macro-redefined -D__DATE__= -D__TIME__= -D__TIMESTAMP__= -fcolor-diagnostics -arch x86_64 -Wall -Werror -Wextra -Wpartial-availability -Wno-missing-field-initializers -Wno-unused-parameter -Wno-c++11-narrowing -Wno-covered-switch-default -Wno-unneeded-internal-declaration -Wno-inconsistent-missing-override -Wno-undefined-var-template -Wno-nonportable-include-path -Wno-address-of-packed-member -Wno-unused-lambda-capture -Wno-user-defined-warnings -O2 -fno-omit-frame-pointer -g1 -isysroot ../../build/mac_files/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.10.sdk -mmacosx-version-min=10.9 -fvisibility=hidden -Xclang -load -Xclang ../../third_party/llvm-build/Release+Asserts/lib/libFindBadConstructs.dylib -Xclang -add-plugin -Xclang find-bad-constructs -Xclang -plugin-arg-find-bad-constructs -Xclang check-auto-raw-pointer -Wheader-hygiene -Wstring-conversion -Wtautological-overlap-compare -Wexit-time-destructors -Xclang -load -Xclang ../../third_party/llvm-build/Release+Asserts/lib/libBlinkGCPlugin.dylib -Xclang -add-plugin -Xclang blink-gc-plugin -Xclang -plugin-arg-blink-gc-plugin -Xclang use-chromium-style-naming -Wglobal-constructors -DLIBXML_STATIC= -std=c++11 -stdlib=libc++ -fno-rtti -fno-exceptions -fvisibility-inlines-hidden -c ../../third_party/WebKit/Source/core/css/PropertyRegistry.cpp -o obj/third_party/WebKit/Source/core/css/css_1/PropertyRegistry.o GOMA:clang++ compiling ../../third_party/WebKit/Source/core/css/PropertyRegistry.cpp:*ERROR*: compiler_proxy:2290ms: reached max number of active fail fallbacks [2087/4430] CXX obj/third_party/WebKit/Source/modules/accessibility/accessibility/AXListBoxOption.o Can someone help me debug this? I don't see much data or information to really dive deeper.
,
Jun 29 2017
I know 2 places with Goma execution details: 1. compiler_proxy_log: See postprocess_for_goma.upload_log step. Doesn't seem to have much useful information in this case :( 2. Live logging via cloudtail. It is associated with the machine: https://pantheon.corp.google.com/logs/viewer?project=goma-logs&organizationId=433637338589&minLogLevel=0&expandAll=false&resource=gce_instance%2Finstance_id%2Fbuild202-m4.golo.chromium.org The log has a lot of stuff... We can probably compare it to the log from Buildbot machine and see what's broken exactly.
,
Jun 29 2017
More logs: request: total=136 success=26 failure=3 compiler_proxy: fail=3 compiler_info: stores=0 store_dups=0 miss=0 fail=0 goma: finished=133 cache_hit=23 local_cachehit=0 aborted=0 retry=554 fail=110 fallback_in_setup: parse_fail=0 no_remote=0 http_disabled=0 compiler_info_fail=0 compiler_disabled=0 requested_by_user=0 local: run=110 killed=0 finished=3 local run reason: fail in file request=107 slow goma, local run started in FILE_REQ=3 error: user: E:1 compiler_proxy:936ms: reached max number of active fail fallbacks E:1 compiler_proxy:1116ms: reached max number of active fail fallbacks E:1 compiler_proxy:1080ms: reached max number of active fail fallbacks log: E=248301 W=0 files: requested=70109 uploaded=250115 missed=0 outputs: files=46 rename=0 buf=46 peak_req=8287172 memory: consuming=1879683072 time: uptime=145 include_processor: total=27899 skipped=19501 total_wait_time=128 total_run_time=1993 includecache: entries=896 cache_size=1451090 hit=7801 missed=597 updated=4 evicted=0 orig_total=21648830 orig_max=5493857 orig_ave=24161 filter_total=1451090 filter_max=582022 filter_ave=1619 depscache: table_size=31859 max=2965 total=19730825 average=619 idtable=71801 hit=121 updated=1 missed=14 http_rpc: query=25 retry=0 timeout=0 error=0 https://luci-logdog.appspot.com/v/?s=chromium%2Fbuildbucket%2Fcr-buildbucket.appspot.com%2F8975462125586410080%2F%2B%2Fsteps%2Fpostprocess_for_goma%2F0%2Fsteps%2Fgoma_stat%2F0%2Fstdout
,
Jun 29 2017
The logs vadim linked to are more useful https://screenshot.googleplex.com/tj1SUHvdkOC
,
Jun 29 2017
|
||
►
Sign in to add a comment |
||
Comment 1 by d...@chromium.org
, Jun 29 2017