JUnit version 4.13.2
.00:27:42.039 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1783470462031
00:27:42.103 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
00:27:42.136 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
00:27:42.145 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskwJm9or/test-tmp
00:27:42.167 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
00:27:42.169 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskwJm9or/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
00:27:42.346 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

00:27:42.534 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:27:42.535 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

00:27:42.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:42.538703  1172 mini_oidc.cc:150] Starting JWKS server
00:27:42.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:42.542336  1172 webserver.cc:534] Webserver started at http://127.0.0.1:35107/ using document root <none> and password file <none>
00:27:42.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:42.542721  1172 mini_oidc.cc:186] Starting OIDC Discovery server
00:27:42.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:42.544723  1172 webserver.cc:534] Webserver started at http://127.0.0.1:38645/ using document root <none> and password file <none>
00:27:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T00:27:42Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
00:27:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T00:27:42Z Disabled control of system clock
00:27:42.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:42.599692  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:42.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:42.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/wal
00:27:42.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/data
00:27:42.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:27:42.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:27:42.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:27:42.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:42.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:42.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:42.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/logs
00:27:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/data/info.pb
00:27:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:27:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:27:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:27:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:27:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:27:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:27:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:27:42.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:27:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.62:40671
00:27:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.62
00:27:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:27:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:27:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:27:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35107/jwks.json with env {}
00:27:42.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.887508  1184 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:27:42.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.888047  1184 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:27:42.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.888530  1184 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:27:42.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.919031  1184 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:27:42.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.919296  1184 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:27:42.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.919498  1184 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:27:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.919688  1184 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:27:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.919899  1184 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:27:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.920287  1184 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35107/jwks.json
00:27:42.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:42.957841  1184 master_runner.cc:387] Master server non-default flags:
00:27:42.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:27:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/data
00:27:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/wal
00:27:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:27:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:27:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.62:40671
00:27:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:27:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:27:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:27:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35107/jwks.json
00:27:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/data/info.pb
00:27:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.62
00:27:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:27:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:27:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:27:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/logs
00:27:42.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:42.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:27:42.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:27:42.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:27:42.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:27:42.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:27:42.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:27:42.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:27:42.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:27:42.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:27:42.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:42.959146  1184 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:27:42.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:42.960942  1184 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:27:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.971148  1190 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:44.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.374701  1189 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 1184
00:27:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.502363  1184 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.530s	user 0.627s	sys 0.903s
00:27:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.502641  1184 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.531s	user 0.627s	sys 0.903s
00:27:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:42.972978  1191 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.505134  1193 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.508169  1192 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1534 milliseconds
00:27:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.508191  1184 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:27:44.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.509557  1184 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:27:44.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.511909  1184 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:27:44.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.513252  1184 hybrid_clock.cc:648] HybridClock initialized: now 1783470464513213 us; error 43 us; skew 500 ppm
00:27:44.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.514129  1184 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:27:44.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.519896  1184 webserver.cc:534] Webserver started at http://127.1.37.62:33021/ using document root <none> and password file <none>
00:27:44.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.520884  1184 fs_manager.cc:362] Metadata directory not provided
00:27:44.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.521092  1184 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:27:44.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.521672  1184 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:27:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.525939  1184 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/data/instance:
00:27:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8f0ce82376c44b39376c23f3a99434f"
00:27:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:44 on dist-test-slave-4l7f"
00:27:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.527287  1184 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/wal/instance:
00:27:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8f0ce82376c44b39376c23f3a99434f"
00:27:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:44 on dist-test-slave-4l7f"
00:27:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.534184  1184 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.004s
00:27:44.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.539618  1200 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:27:44.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.540652  1184 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
00:27:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.540983  1184 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/wal
00:27:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8f0ce82376c44b39376c23f3a99434f"
00:27:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:44 on dist-test-slave-4l7f"
00:27:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.541409  1184 fs_report.cc:389] FS layout report
00:27:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:27:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/wal
00:27:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/wal
00:27:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/data/data
00:27:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:27:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:27:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:27:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:27:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:27:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:27:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:27:44.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:27:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:27:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:27:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:27:44.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.594148  1184 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:27:44.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.595779  1184 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:27:44.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.596271  1184 kserver.cc:163] Server-wide thread pool size limit: 3276
00:27:44.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.663515  1184 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.62:40671
00:27:44.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.663575  1251 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.62:40671 every 8 connection(s)
00:27:44.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.666312  1184 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/data/info.pb
00:27:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.671900  1252 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:27:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.673380  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 1184
00:27:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.673888  1172 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-0/wal/instance
00:27:44.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.678776  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:44.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:44.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/wal
00:27:44.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/data
00:27:44.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:27:44.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:27:44.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:27:44.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:44.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/logs
00:27:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/data/info.pb
00:27:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:27:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:27:44.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:27:44.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:27:44.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:44.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:44.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:44.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:27:44.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:27:44.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:27:44.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:27:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.61:35087
00:27:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.61
00:27:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:27:44.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:27:44.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:44.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:27:44.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35107/jwks.json with env {}
00:27:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.686012  1252 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } has no permanent_uuid. Determining permanent_uuid...
00:27:44.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.714082  1252 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } has no permanent_uuid. Determining permanent_uuid...
00:27:44.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.716648  1203 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.61:35087: connect: Connection refused (error 111)
00:27:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.720573  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.61:35087: Network error: Client connection negotiation failed: client connection to 127.1.37.61:35087: connect: Connection refused (error 111)
00:27:44.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.770124  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } attempt: 1
00:27:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.775120  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.61:35087: Network error: Client connection negotiation failed: client connection to 127.1.37.61:35087: connect: Connection refused (error 111)
00:27:44.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.843488  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } attempt: 2
00:27:44.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.847625  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.61:35087: Network error: Client connection negotiation failed: client connection to 127.1.37.61:35087: connect: Connection refused (error 111)
00:27:44.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:44.938978  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } attempt: 3
00:27:44.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.942998  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.61:35087: Network error: Client connection negotiation failed: client connection to 127.1.37.61:35087: connect: Connection refused (error 111)
00:27:44.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.980796  1254 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:27:44.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.981376  1254 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:27:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:44.981987  1254 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:27:45.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.013161  1254 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:27:45.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.013425  1254 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:27:45.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.013614  1254 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:27:45.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.013799  1254 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:27:45.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.014086  1254 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:27:45.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.014339  1254 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35107/jwks.json
00:27:45.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:45.050209  1254 master_runner.cc:387] Master server non-default flags:
00:27:45.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:27:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/data
00:27:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/wal
00:27:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:27:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:27:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.61:35087
00:27:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:27:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:27:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:27:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35107/jwks.json
00:27:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/data/info.pb
00:27:45.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.61
00:27:45.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:45.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:27:45.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:45.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:27:45.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:27:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/logs
00:27:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:27:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:27:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:27:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:27:45.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:27:45.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:27:45.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:27:45.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:27:45.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:27:45.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:45.051561  1254 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:27:45.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:45.053308  1254 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:27:45.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.064867  1263 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:45.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:45.086330  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } attempt: 4
00:27:45.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.090286  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.61:35087: Network error: Client connection negotiation failed: client connection to 127.1.37.61:35087: connect: Connection refused (error 111)
00:27:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:45.389788  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } attempt: 5
00:27:45.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.395088  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.61:35087: Network error: Client connection negotiation failed: client connection to 127.1.37.61:35087: connect: Connection refused (error 111)
00:27:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:45.942448  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } attempt: 6
00:27:45.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.949554  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.61:35087: Network error: Client connection negotiation failed: client connection to 127.1.37.61:35087: connect: Connection refused (error 111)
00:27:46.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.070464  1266 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:46.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:45.066308  1264 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:46.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.171756  1265 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
00:27:46.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.171876  1254 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:27:46.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.173501  1254 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:27:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.176620  1254 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:27:46.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.178033  1254 hybrid_clock.cc:648] HybridClock initialized: now 1783470466177979 us; error 61 us; skew 500 ppm
00:27:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.179361  1254 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:27:46.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.185017  1254 webserver.cc:534] Webserver started at http://127.1.37.61:39647/ using document root <none> and password file <none>
00:27:46.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.186095  1254 fs_manager.cc:362] Metadata directory not provided
00:27:46.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.186307  1254 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:27:46.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.186883  1254 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:27:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.191112  1254 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/data/instance:
00:27:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae4e256058884d839e8f542ffc87766e"
00:27:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:46 on dist-test-slave-4l7f"
00:27:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.192279  1254 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/wal/instance:
00:27:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae4e256058884d839e8f542ffc87766e"
00:27:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:46 on dist-test-slave-4l7f"
00:27:46.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.198953  1254 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.000s
00:27:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.204069  1275 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:27:46.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.205132  1254 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:27:46.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.205451  1254 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/wal
00:27:46.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae4e256058884d839e8f542ffc87766e"
00:27:46.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:46 on dist-test-slave-4l7f"
00:27:46.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.205813  1254 fs_report.cc:389] FS layout report
00:27:46.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:27:46.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/wal
00:27:46.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/wal
00:27:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/data/data
00:27:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:27:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:27:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:27:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:27:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:27:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:27:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:27:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:27:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:27:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:27:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:27:46.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.263589  1254 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:27:46.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.265233  1254 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:27:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.265735  1254 kserver.cc:163] Server-wide thread pool size limit: 3276
00:27:46.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.333449  1254 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.61:35087
00:27:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.333520  1326 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.61:35087 every 8 connection(s)
00:27:46.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.336319  1254 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/data/info.pb
00:27:46.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.342658  1327 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:27:46.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.345623  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 1254
00:27:46.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.345947  1172 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-1/wal/instance
00:27:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.347898  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/wal
00:27:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/data
00:27:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:27:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:27:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:27:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/logs
00:27:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/data/info.pb
00:27:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:27:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:27:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:27:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:27:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:27:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:27:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:27:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:27:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.60:42571
00:27:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.60
00:27:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:27:46.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:27:46.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:46.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:27:46.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35107/jwks.json with env {}
00:27:46.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.355262  1327 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } has no permanent_uuid. Determining permanent_uuid...
00:27:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.375120  1327 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } has no permanent_uuid. Determining permanent_uuid...
00:27:46.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.389308  1327 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } has no permanent_uuid. Determining permanent_uuid...
00:27:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.391466  1278 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.395395  1327 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:46.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.444898  1327 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 1
00:27:46.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.448944  1327 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:46.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.517386  1327 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 2
00:27:46.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.521431  1327 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:46.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.612855  1327 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 3
00:27:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.617249  1327 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:46.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.647070  1329 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:27:46.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.647635  1329 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:27:46.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.648072  1329 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:27:46.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.679394  1329 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
00:27:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.679677  1329 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:27:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.679870  1329 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
00:27:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.680087  1329 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
00:27:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.680275  1329 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
00:27:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.680460  1329 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35107/jwks.json
00:27:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.715791  1329 master_runner.cc:387] Master server non-default flags:
00:27:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:27:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/data
00:27:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/wal
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.60:42571
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35107/jwks.json
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/data/info.pb
00:27:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.60
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/logs
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:27:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:27:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:27:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:27:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:27:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:27:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.717077  1329 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:27:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.719162  1329 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:27:46.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.730376  1339 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:46.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:46.760638  1327 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 4
00:27:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.764981  1327 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:47.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.010069  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } attempt: 7
00:27:47.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.037940  1252 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } has no permanent_uuid. Determining permanent_uuid...
00:27:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:47.044983  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:47.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.064343  1327 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 5
00:27:47.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:47.070854  1327 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:47.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.103482  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 1
00:27:47.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:47.108819  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.190318  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 2
00:27:47.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:47.195668  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:47.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.281188  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 3
00:27:47.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:47.286675  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:47.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.427232  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 4
00:27:47.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:47.432840  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:47.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.618350  1327 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 6
00:27:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:47.627456  1327 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:47.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.716286  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 5
00:27:47.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:47.724090  1252 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.37.60:42571: Network error: Client connection negotiation failed: client connection to 127.1.37.60:42571: connect: Connection refused (error 111)
00:27:47.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:46.732211  1340 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:47.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:47.874914  1342 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:47.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:47.877364  1341 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1141 milliseconds
00:27:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.877511  1329 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:27:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.878728  1329 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:27:47.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.881162  1329 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:27:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.882498  1329 hybrid_clock.cc:648] HybridClock initialized: now 1783470467882423 us; error 75 us; skew 500 ppm
00:27:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.883234  1329 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:27:47.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.889144  1329 webserver.cc:534] Webserver started at http://127.1.37.60:34573/ using document root <none> and password file <none>
00:27:47.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.890025  1329 fs_manager.cc:362] Metadata directory not provided
00:27:47.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.890204  1329 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:27:47.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.890583  1329 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:27:47.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.894709  1329 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/data/instance:
00:27:47.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "599c4622b70f4f55a19079a449c8423e"
00:27:47.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:47 on dist-test-slave-4l7f"
00:27:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.895747  1329 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/wal/instance:
00:27:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "599c4622b70f4f55a19079a449c8423e"
00:27:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:47 on dist-test-slave-4l7f"
00:27:47.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.902495  1329 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.009s	sys 0.000s
00:27:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.907814  1355 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:27:47.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.910143  1329 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.000s
00:27:47.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.910482  1329 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/wal
00:27:47.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "599c4622b70f4f55a19079a449c8423e"
00:27:47.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:47 on dist-test-slave-4l7f"
00:27:47.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.910851  1329 fs_report.cc:389] FS layout report
00:27:47.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:27:47.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/wal
00:27:47.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/wal
00:27:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/data/data
00:27:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:27:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:27:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:27:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:27:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:27:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:27:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:27:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:27:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:27:47.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:27:47.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:27:47.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.962980  1329 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:27:47.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.964735  1329 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:27:47.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:47.965252  1329 kserver.cc:163] Server-wide thread pool size limit: 3276
00:27:48.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.033550  1329 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.60:42571
00:27:48.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.033631  1406 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.60:42571 every 8 connection(s)
00:27:48.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.036337  1329 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/data/info.pb
00:27:48.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.040948  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 1329
00:27:48.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.041268  1172 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/master-2/wal/instance
00:27:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.042873  1407 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:27:48.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.055698  1407 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } has no permanent_uuid. Determining permanent_uuid...
00:27:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.078433  1407 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } has no permanent_uuid. Determining permanent_uuid...
00:27:48.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.087451  1407 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } has no permanent_uuid. Determining permanent_uuid...
00:27:48.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.102680  1407 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e: Bootstrap starting.
00:27:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.109184  1407 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e: Neither blocks nor log segments found. Creating new log.
00:27:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.110805  1407 log.cc:826] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e: Log is configured to *not* fsync() on all Append() calls
00:27:48.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.116010  1407 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e: No bootstrap required, opened a new log
00:27:48.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.134142  1407 raft_consensus.cc:359] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } }
00:27:48.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.134945  1407 raft_consensus.cc:385] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:27:48.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.135210  1407 raft_consensus.cc:740] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 599c4622b70f4f55a19079a449c8423e, State: Initialized, Role: FOLLOWER
00:27:48.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.136018  1407 consensus_queue.cc:260] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } }
00:27:48.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.138371  1416 sys_catalog.cc:455] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } } }
00:27:48.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.138926  1416 sys_catalog.cc:458] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [sys.catalog]: This master's current role is: FOLLOWER
00:27:48.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.139761  1407 sys_catalog.cc:565] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [sys.catalog]: configured and running, proceeding with master startup.
00:27:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:48.167461  1427 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:27:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:48.167810  1427 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:27:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.162067  1329 master_runner.cc:187] Error getting master registration for 127.1.37.62:40671: OK, instance_id { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" instance_seqno: 1783470464644872 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:27:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.182514  1329 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:27:48.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:48.220469  1361 tablet.cc:2406] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:27:48.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.276547  1252 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 6
00:27:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.293548  1252 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f: Bootstrap starting.
00:27:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.299872  1252 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f: Neither blocks nor log segments found. Creating new log.
00:27:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.301587  1252 log.cc:826] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f: Log is configured to *not* fsync() on all Append() calls
00:27:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.305629  1252 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f: No bootstrap required, opened a new log
00:27:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.323830  1252 raft_consensus.cc:359] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } }
00:27:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.324436  1252 raft_consensus.cc:385] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:27:48.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.324641  1252 raft_consensus.cc:740] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8f0ce82376c44b39376c23f3a99434f, State: Initialized, Role: FOLLOWER
00:27:48.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.325315  1252 consensus_queue.cc:260] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } }
00:27:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.327664  1431 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } } }
00:27:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.328480  1431 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [sys.catalog]: This master's current role is: FOLLOWER
00:27:48.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.329321  1252 sys_catalog.cc:565] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [sys.catalog]: configured and running, proceeding with master startup.
00:27:48.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:48.355360  1206 tablet.cc:2406] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:27:48.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:48.373030  1442 catalog_manager.cc:1593] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:27:48.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:48.373869  1442 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:27:48.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.366158  1184 master_runner.cc:187] Error getting master registration for 127.1.37.61:35087: OK, instance_id { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" instance_seqno: 1783470466315796 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:27:48.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.387912  1184 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
00:27:48.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.448983  1416 raft_consensus.cc:493] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:27:48.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.449551  1416 raft_consensus.cc:515] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } }
00:27:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.453308  1227 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "599c4622b70f4f55a19079a449c8423e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8f0ce82376c44b39376c23f3a99434f" is_pre_election: true
00:27:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.453583  1416 leader_election.cc:290] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a8f0ce82376c44b39376c23f3a99434f (127.1.37.62:40671), ae4e256058884d839e8f542ffc87766e (127.1.37.61:35087)
00:27:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.454206  1227 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 599c4622b70f4f55a19079a449c8423e in term 0.
00:27:48.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.455634  1359 leader_election.cc:304] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 599c4622b70f4f55a19079a449c8423e, a8f0ce82376c44b39376c23f3a99434f; no voters: 
00:27:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.456570  1416 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 0 FOLLOWER]: Leader pre-election won for term 1
00:27:48.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.456921  1416 raft_consensus.cc:493] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:27:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.457235  1416 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 0 FOLLOWER]: Advancing to term 1
00:27:48.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.467345  1416 raft_consensus.cc:515] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } }
00:27:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.478022  1227 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "599c4622b70f4f55a19079a449c8423e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8f0ce82376c44b39376c23f3a99434f"
00:27:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.478564  1227 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [term 0 FOLLOWER]: Advancing to term 1
00:27:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.485869  1227 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 599c4622b70f4f55a19079a449c8423e in term 1.
00:27:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.486986  1359 leader_election.cc:304] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 599c4622b70f4f55a19079a449c8423e, a8f0ce82376c44b39376c23f3a99434f; no voters: 
00:27:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.489674  1444 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 1 FOLLOWER]: Leader election won for term 1
00:27:48.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.491318  1416 leader_election.cc:290] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [CANDIDATE]: Term 1 election: Requested vote from peers a8f0ce82376c44b39376c23f3a99434f (127.1.37.62:40671), ae4e256058884d839e8f542ffc87766e (127.1.37.61:35087)
00:27:48.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.493477  1444 raft_consensus.cc:697] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [term 1 LEADER]: Becoming Leader. State: Replica: 599c4622b70f4f55a19079a449c8423e, State: Running, Role: LEADER
00:27:48.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.494638  1444 consensus_queue.cc:237] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } }
00:27:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.506337  1416 sys_catalog.cc:455] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 599c4622b70f4f55a19079a449c8423e. Latest consensus state: current_term: 1 leader_uuid: "599c4622b70f4f55a19079a449c8423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } } }
00:27:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.509903  1416 sys_catalog.cc:458] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [sys.catalog]: This master's current role is: LEADER
00:27:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.513204  1447 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
00:27:48.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.521770  1447 catalog_manager.cc:1519] Initializing Kudu cluster ID...
00:27:48.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.541227  1227 raft_consensus.cc:1275] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [term 1 FOLLOWER]: Refusing update from remote peer 599c4622b70f4f55a19079a449c8423e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:27:48.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.545791  1416 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:27:48.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.569723  1431 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 599c4622b70f4f55a19079a449c8423e. Latest consensus state: current_term: 1 leader_uuid: "599c4622b70f4f55a19079a449c8423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } } }
00:27:48.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.570406  1431 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [sys.catalog]: This master's current role is: FOLLOWER
00:27:48.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.584071  1416 sys_catalog.cc:455] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "599c4622b70f4f55a19079a449c8423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } } }
00:27:48.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.584780  1416 sys_catalog.cc:458] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [sys.catalog]: This master's current role is: LEADER
00:27:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.585988  1447 catalog_manager.cc:1382] Generated new cluster ID: 1e7acff08acb40cba822a2cce0e5ca06
00:27:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.586253  1447 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
00:27:48.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.590224  1449 mvcc.cc:204] Tried to move back new op lower bound from 7305095039124672512 to 7305095038967013376. Current Snapshot: MvccSnapshot[applied={T|T < 7305095039124672512}]
00:27:48.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.592648  1431 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "599c4622b70f4f55a19079a449c8423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } } }
00:27:48.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.593583  1431 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f [sys.catalog]: This master's current role is: FOLLOWER
00:27:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.636664  1447 catalog_manager.cc:1405] Generated new certificate authority record
00:27:48.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.640466  1447 catalog_manager.cc:1539] Loading token signing keys...
00:27:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.673828  1447 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e: Generated new TSK 0
00:27:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.674952  1447 catalog_manager.cc:1549] Initializing in-progress tserver states...
00:27:48.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.687868  1327 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } attempt: 7
00:27:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.689687  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:27:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data
00:27:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:27:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:27:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/logs
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.1:0
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.1
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.1
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:27:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.735198  1327 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e: Bootstrap starting.
00:27:48.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.755457  1327 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e: Neither blocks nor log segments found. Creating new log.
00:27:48.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.770555  1327 log.cc:826] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e: Log is configured to *not* fsync() on all Append() calls
00:27:48.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.783802  1327 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e: No bootstrap required, opened a new log
00:27:48.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.851356  1327 raft_consensus.cc:359] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } }
00:27:48.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.853334  1327 raft_consensus.cc:385] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:27:48.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.854032  1327 raft_consensus.cc:740] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ae4e256058884d839e8f542ffc87766e, State: Initialized, Role: FOLLOWER
00:27:48.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.857271  1327 consensus_queue.cc:260] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } }
00:27:48.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.865504  1456 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } } }
00:27:48.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.866340  1456 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [sys.catalog]: This master's current role is: FOLLOWER
00:27:48.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.871142  1301 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "599c4622b70f4f55a19079a449c8423e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ae4e256058884d839e8f542ffc87766e"
00:27:48.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.871865  1301 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [term 0 FOLLOWER]: Advancing to term 1
00:27:48.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.874737  1302 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "599c4622b70f4f55a19079a449c8423e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ae4e256058884d839e8f542ffc87766e" is_pre_election: true
00:27:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.882038  1301 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 599c4622b70f4f55a19079a449c8423e in term 1.
00:27:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.883925  1300 raft_consensus.cc:1275] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [term 1 FOLLOWER]: Refusing update from remote peer 599c4622b70f4f55a19079a449c8423e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:27:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.890381  1444 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [LEADER]: Connected to new peer: Peer: permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:27:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.900920  1327 sys_catalog.cc:565] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [sys.catalog]: configured and running, proceeding with master startup.
00:27:48.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.963529  1456 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 599c4622b70f4f55a19079a449c8423e. Latest consensus state: current_term: 1 leader_uuid: "599c4622b70f4f55a19079a449c8423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } } }
00:27:48.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.963678  1462 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "599c4622b70f4f55a19079a449c8423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } } }
00:27:48.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.964257  1456 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [sys.catalog]: This master's current role is: FOLLOWER
00:27:48.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.964337  1462 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e [sys.catalog]: This master's current role is: FOLLOWER
00:27:48.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:48.993206  1472 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
00:27:49.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:49.217779  1454 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:27:49.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:49.218443  1454 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:27:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:49.219165  1454 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:27:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:49.276118  1454 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:27:49.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:49.277482  1454 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.1
00:27:49.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:49.340236  1454 tablet_server_runner.cc:78] Tablet server non-default flags:
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.1:0
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:27:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.1
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.1
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/logs
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:27:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:27:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:27:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:49.344518  1454 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:27:49.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:49.346911  1454 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:27:49.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:49.379848  1442 catalog_manager.cc:1294] Loaded cluster ID: 1e7acff08acb40cba822a2cce0e5ca06
00:27:49.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:49.380576  1442 catalog_manager.cc:1587] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f: loading cluster ID for follower catalog manager: success
00:27:49.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:49.388064  1442 catalog_manager.cc:1609] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f: acquiring CA information for follower catalog manager: success
00:27:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:49.394878  1442 catalog_manager.cc:1637] T 00000000000000000000000000000000 P a8f0ce82376c44b39376c23f3a99434f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:27:49.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:49.398047  1478 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:49.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:49.472196  1444 sys_catalog.cc:455] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "599c4622b70f4f55a19079a449c8423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8f0ce82376c44b39376c23f3a99434f" member_type: VOTER last_known_addr { host: "127.1.37.62" port: 40671 } } peers { permanent_uuid: "ae4e256058884d839e8f542ffc87766e" member_type: VOTER last_known_addr { host: "127.1.37.61" port: 35087 } } peers { permanent_uuid: "599c4622b70f4f55a19079a449c8423e" member_type: VOTER last_known_addr { host: "127.1.37.60" port: 42571 } } }
00:27:49.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:49.473362  1444 sys_catalog.cc:458] T 00000000000000000000000000000000 P 599c4622b70f4f55a19079a449c8423e [sys.catalog]: This master's current role is: LEADER
00:27:50.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:50.058200  1472 catalog_manager.cc:1294] Loaded cluster ID: 1e7acff08acb40cba822a2cce0e5ca06
00:27:50.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:50.058488  1472 catalog_manager.cc:1587] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e: loading cluster ID for follower catalog manager: success
00:27:50.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:50.072194  1472 catalog_manager.cc:1609] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e: acquiring CA information for follower catalog manager: success
00:27:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:50.103953  1472 catalog_manager.cc:1637] T 00000000000000000000000000000000 P ae4e256058884d839e8f542ffc87766e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:27:51.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:50.772452  1477 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 1454
00:27:51.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:49.403224  1479 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:51.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:51.405923  1454 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.002s	user 0.001s	sys 0.000s
00:27:51.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:51.406240  1454 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.003s	user 0.001s	sys 0.000s
00:27:51.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:51.407884  1482 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:51.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:51.410634  1480 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2001 milliseconds
00:27:51.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.410729  1454 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:27:51.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.412140  1454 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:27:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.414675  1454 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:27:51.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.416297  1454 hybrid_clock.cc:648] HybridClock initialized: now 1783470471416259 us; error 32 us; skew 500 ppm
00:27:51.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.417419  1454 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:27:51.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.424610  1454 webserver.cc:534] Webserver started at http://127.1.37.1:44645/ using document root <none> and password file <none>
00:27:51.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.425741  1454 fs_manager.cc:362] Metadata directory not provided
00:27:51.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.426007  1454 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:27:51.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.426533  1454 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:27:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.432623  1454 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/instance:
00:27:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0f7154f76e840bc92259656e7fd7d0b"
00:27:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:51 on dist-test-slave-4l7f"
00:27:51.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.434008  1454 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal/instance:
00:27:51.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0f7154f76e840bc92259656e7fd7d0b"
00:27:51.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:51 on dist-test-slave-4l7f"
00:27:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.442806  1454 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.009s	sys 0.000s
00:27:51.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.449852  1491 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:27:51.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.450925  1454 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
00:27:51.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.451187  1454 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:27:51.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0f7154f76e840bc92259656e7fd7d0b"
00:27:51.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:51 on dist-test-slave-4l7f"
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.451514  1454 fs_report.cc:389] FS layout report
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/data
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:27:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:27:51.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:27:51.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:27:51.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:27:51.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.535887  1454 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:27:51.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.537470  1454 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:27:51.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.537967  1454 kserver.cc:163] Server-wide thread pool size limit: 3276
00:27:51.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.540797  1454 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:27:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.545097  1454 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:27:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.545305  1454 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:27:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.545673  1454 ts_tablet_manager.cc:616] Registered 0 tablets
00:27:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.545861  1454 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:27:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.706277  1454 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.1:36133
00:27:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.706459  1603 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.1:36133 every 8 connection(s)
00:27:51.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.708878  1454 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:27:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.711674  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 1454
00:27:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.712111  1172 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal/instance
00:27:51.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.727365  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/logs
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:27:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:27:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.2:0
00:27:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.2
00:27:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.2
00:27:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:51.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:51.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:51.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:27:51.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.773798  1608 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:27:51.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.774297  1608 heartbeater.cc:461] Registering TS with master...
00:27:51.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.774471  1604 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:27:51.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.774736  1604 heartbeater.cc:461] Registering TS with master...
00:27:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.776542  1604 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:27:51.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.776567  1608 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:27:51.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.777410  1605 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:27:51.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.777724  1605 heartbeater.cc:461] Registering TS with master...
00:27:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.778555  1605 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:27:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.780373  1216 ts_manager.cc:194] Registered new tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:27:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.781041  1372 ts_manager.cc:194] Registered new tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:27:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.781493  1292 ts_manager.cc:194] Registered new tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:27:51.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:51.783597  1372 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.1:49669
00:27:52.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:52.064163  1613 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:27:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:52.064860  1613 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:27:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:52.065451  1613 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:27:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:52.098272  1613 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:27:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:52.099234  1613 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.2
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:52.134627  1613 tablet_server_runner.cc:78] Tablet server non-default flags:
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.2:0
00:27:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.2
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.2
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/logs
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:27:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:52.136303  1613 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:27:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:52.138525  1613 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:27:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:52.151373  1620 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:52.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:52.787196  1604 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:27:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:53.553996  1619 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 1613
00:27:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:53.558190  1619 kernel_stack_watchdog.cc:198] Thread 1613 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 399ms:
00:27:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
00:27:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
00:27:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
00:27:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
00:27:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:52.153223  1621 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:53.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:53.558714  1613 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.406s	user 0.499s	sys 0.800s
00:27:53.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:53.559051  1613 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.406s	user 0.499s	sys 0.800s
00:27:53.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:53.561008  1623 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:53.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:53.563464  1622 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1405 milliseconds
00:27:53.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.563488  1613 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:27:53.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.564713  1613 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:27:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.566727  1613 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:27:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.568068  1613 hybrid_clock.cc:648] HybridClock initialized: now 1783470473568033 us; error 31 us; skew 500 ppm
00:27:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.568892  1613 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:27:53.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.574633  1613 webserver.cc:534] Webserver started at http://127.1.37.2:33733/ using document root <none> and password file <none>
00:27:53.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.575567  1613 fs_manager.cc:362] Metadata directory not provided
00:27:53.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.575781  1613 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:27:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.576283  1613 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:27:53.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.580446  1613 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/instance:
00:27:53.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f84a67fab33464bb93a32fd5b934b8d"
00:27:53.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:53 on dist-test-slave-4l7f"
00:27:53.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.581578  1613 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal/instance:
00:27:53.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f84a67fab33464bb93a32fd5b934b8d"
00:27:53.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:53 on dist-test-slave-4l7f"
00:27:53.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.588373  1613 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.000s
00:27:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.593570  1632 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:27:53.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.594555  1613 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.005s
00:27:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.594869  1613 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:27:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f84a67fab33464bb93a32fd5b934b8d"
00:27:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:53 on dist-test-slave-4l7f"
00:27:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.595212  1613 fs_report.cc:389] FS layout report
00:27:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:27:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:27:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/data
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:27:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:27:53.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.640293  1613 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:27:53.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.641855  1613 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:27:53.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.642340  1613 kserver.cc:163] Server-wide thread pool size limit: 3276
00:27:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.644917  1613 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:27:53.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.649050  1613 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:27:53.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.649255  1613 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:27:53.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.649565  1613 ts_tablet_manager.cc:616] Registered 0 tablets
00:27:53.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.649745  1613 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:27:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.788060  1613 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.2:37155
00:27:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.788208  1744 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.2:37155 every 8 connection(s)
00:27:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.790995  1613 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:27:53.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.793880  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 1613
00:27:53.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.794515  1172 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal/instance
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.809141  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/logs
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:27:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.3:0
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.3
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.3
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:53.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:27:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.839537  1745 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:27:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.840026  1745 heartbeater.cc:461] Registering TS with master...
00:27:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.841400  1745 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:27:53.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.843995  1372 ts_manager.cc:194] Registered new tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:27:53.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.845624  1372 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.2:50541
00:27:53.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.849906  1746 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:27:53.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.850189  1746 heartbeater.cc:461] Registering TS with master...
00:27:53.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.850818  1746 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:27:53.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.853121  1292 ts_manager.cc:194] Registered new tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:27:53.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.853894  1749 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:27:53.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.854228  1749 heartbeater.cc:461] Registering TS with master...
00:27:53.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.854828  1749 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:27:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:53.857342  1216 ts_manager.cc:194] Registered new tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:27:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:54.127405  1755 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:27:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:54.127912  1755 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:27:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:54.128327  1755 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:27:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:54.163775  1755 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:27:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:54.164606  1755 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.3
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:54.199939  1755 tablet_server_runner.cc:78] Tablet server non-default flags:
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.3:0
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.3
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:27:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.3
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/logs
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:27:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:54.201234  1755 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:27:54.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:54.202744  1755 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:27:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:54.214395  1761 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:54.849999  1745 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:27:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:55.617506  1760 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 1755
00:27:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:55.732009  1760 kernel_stack_watchdog.cc:198] Thread 1755 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 400ms:
00:27:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
00:27:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
00:27:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
00:27:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
00:27:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:54.215168  1762 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:55.732741  1755 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.518s	user 0.632s	sys 0.876s
00:27:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:55.733264  1755 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.519s	user 0.632s	sys 0.877s
00:27:55.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:55.735018  1764 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:27:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:55.736958  1763 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1518 milliseconds
00:27:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.736975  1755 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:27:55.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.738171  1755 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:27:55.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.740152  1755 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:27:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.741498  1755 hybrid_clock.cc:648] HybridClock initialized: now 1783470475741459 us; error 39 us; skew 500 ppm
00:27:55.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.742312  1755 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:27:55.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.751880  1755 webserver.cc:534] Webserver started at http://127.1.37.3:44545/ using document root <none> and password file <none>
00:27:55.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.752799  1755 fs_manager.cc:362] Metadata directory not provided
00:27:55.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.753008  1755 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:27:55.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.753499  1755 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
00:27:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.757651  1755 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/instance:
00:27:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:27:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:55 on dist-test-slave-4l7f"
00:27:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.758759  1755 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal/instance:
00:27:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:27:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:55 on dist-test-slave-4l7f"
00:27:55.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.765439  1755 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.002s
00:27:55.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.770658  1772 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:27:55.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.771587  1755 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.005s
00:27:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.771900  1755 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:27:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:27:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:55 on dist-test-slave-4l7f"
00:27:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.772212  1755 fs_report.cc:389] FS layout report
00:27:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:27:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:27:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/data
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:27:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:27:55.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.831509  1755 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:27:55.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.833267  1755 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:27:55.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.833791  1755 kserver.cc:163] Server-wide thread pool size limit: 3276
00:27:55.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.836987  1755 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:27:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.841385  1755 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:27:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.841660  1755 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
00:27:55.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.841928  1755 ts_tablet_manager.cc:616] Registered 0 tablets
00:27:55.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.842111  1755 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
00:27:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.974718  1755 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.3:36659
00:27:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.974828  1884 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.3:36659 every 8 connection(s)
00:27:55.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.977269  1755 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:27:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.982827  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 1755
00:27:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:55.983312  1172 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal/instance
00:27:56.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.015774  1885 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:27:56.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.016278  1885 heartbeater.cc:461] Registering TS with master...
00:27:56.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.017515  1885 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:27:56.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.018083  1887 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:27:56.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.018380  1887 heartbeater.cc:461] Registering TS with master...
00:27:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.019232  1887 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:27:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.019423  1886 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:27:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.019718  1886 heartbeater.cc:461] Registering TS with master...
00:27:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.020134  1372 ts_manager.cc:194] Registered new tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:27:56.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.020565  1886 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:27:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.021797  1372 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.3:41301
00:27:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.023434  1292 ts_manager.cc:194] Registered new tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:27:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.024333  1216 ts_manager.cc:194] Registered new tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:27:56.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.032361  1172 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
00:27:56.043 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
00:27:56.045 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

00:27:56.056 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.1.37.62"
      port: 40671
    }
    bound_http_address {
      host: "127.1.37.62"
      port: 33021
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.1.37.61"
      port: 35087
    }
    bound_http_address {
      host: "127.1.37.61"
      port: 39647
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.1.37.60"
      port: 42571
    }
    bound_http_address {
      host: "127.1.37.60"
      port: 34573
    }
  }
}

00:27:56.059 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

00:27:56.061 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.1.37.1"
      port: 36133
    }
    bound_http_address {
      host: "127.1.37.1"
      port: 44645
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.1.37.2"
      port: 37155
    }
    bound_http_address {
      host: "127.1.37.2"
      port: 33733
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.1.37.3"
      port: 36659
    }
    bound_http_address {
      host: "127.1.37.3"
      port: 44545
    }
  }
}

00:27:56.062 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
00:27:56.218 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
00:27:56.271 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
00:27:56.456 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.37.60:42571(127.1.37.60:42571)] connecting to peer
00:27:56.508 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.37.62:40671(127.1.37.62:40671)] connecting to peer
00:27:56.510 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.37.61:35087(127.1.37.61:35087)] connecting to peer
00:27:56.528 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.37.62:40671(127.1.37.62:40671)] Successfully connected to peer
00:27:56.531 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.1.37.61:35087(127.1.37.61:35087)] Successfully connected to peer
00:27:56.531 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.37.60:42571(127.1.37.60:42571)] Successfully connected to peer
00:27:56.624 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.1.37.61
00:27:56.625 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.1.37.62
00:27:56.626 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.1.37.60
00:27:56.845 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xccdd8693, L:/127.0.0.1:59286 - R:/127.1.37.61:35087] using SASL/PLAIN
00:27:56.847 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x61102084, L:/127.0.0.1:50450 - R:/127.1.37.60:42571] using SASL/PLAIN
00:27:56.849 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa1c7f0e1, L:/127.0.0.1:48790 - R:/127.1.37.62:40671] using SASL/PLAIN
00:27:56.882 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
00:27:56.883 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=460), Traces: [0ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.37.60:42571, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.37.62:40671, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.37.61:35087, [435ms] Sub RPC ConnectToMaster: received response from server master-127.1.37.61:35087: OK, [435ms] Sub RPC ConnectToMaster: received response from server master-127.1.37.62:40671: OK, [435ms] Sub RPC ConnectToMaster: received response from server master-127.1.37.60:42571: OK, deferred=null) after lookup
00:27:56.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:56.932670  1372 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50450:
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407"
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
00:27:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
00:27:56.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
00:27:56.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
00:27:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:56.936717  1372 catalog_manager.cc:7076] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
00:27:57.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.026021  1885 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:27:57.029 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table be871dc169c84c79815b569640956aeb
00:27:57.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.032526  1820 tablet_service.cc:1511] Processing CreateTablet for tablet 5209d2744dd54173958e67ef6c703429 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407 [id=be871dc169c84c79815b569640956aeb]), partition=RANGE (key) PARTITION UNBOUNDED
00:27:57.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.034615  1820 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5209d2744dd54173958e67ef6c703429. 1 dirs total, 0 dirs full, 0 dirs failed
00:27:57.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.037096  1680 tablet_service.cc:1511] Processing CreateTablet for tablet 5209d2744dd54173958e67ef6c703429 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407 [id=be871dc169c84c79815b569640956aeb]), partition=RANGE (key) PARTITION UNBOUNDED
00:27:57.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.038898  1680 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5209d2744dd54173958e67ef6c703429. 1 dirs total, 0 dirs full, 0 dirs failed
00:27:57.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.050653  1539 tablet_service.cc:1511] Processing CreateTablet for tablet 5209d2744dd54173958e67ef6c703429 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407 [id=be871dc169c84c79815b569640956aeb]), partition=RANGE (key) PARTITION UNBOUNDED
00:27:57.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.055542  1539 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5209d2744dd54173958e67ef6c703429. 1 dirs total, 0 dirs full, 0 dirs failed
00:27:57.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.080718  1921 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Bootstrap starting.
00:27:57.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.096064  1921 tablet_bootstrap.cc:654] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Neither blocks nor log segments found. Creating new log.
00:27:57.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.102662  1921 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Log is configured to *not* fsync() on all Append() calls
00:27:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.118548  1922 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap starting.
00:27:57.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.121754  1923 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Bootstrap starting.
00:27:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.125824  1921 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: No bootstrap required, opened a new log
00:27:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.126344  1921 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Time spent bootstrapping tablet: real 0.046s	user 0.011s	sys 0.003s
00:27:57.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.133430  1922 tablet_bootstrap.cc:654] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Neither blocks nor log segments found. Creating new log.
00:27:57.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.134387  1923 tablet_bootstrap.cc:654] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Neither blocks nor log segments found. Creating new log.
00:27:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.136063  1922 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Log is configured to *not* fsync() on all Append() calls
00:27:57.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.146291  1923 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Log is configured to *not* fsync() on all Append() calls
00:27:57.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.152417  1922 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: No bootstrap required, opened a new log
00:27:57.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.153137  1922 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Time spent bootstrapping tablet: real 0.035s	user 0.012s	sys 0.009s
00:27:57.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.153262  1921 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:57.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.154101  1921 raft_consensus.cc:385] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:27:57.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.154405  1921 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d7f44bee9e9544549e3b9aebe7adbdfe, State: Initialized, Role: FOLLOWER
00:27:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.155419  1921 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.163451  1923 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: No bootstrap required, opened a new log
00:27:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.163966  1923 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Time spent bootstrapping tablet: real 0.043s	user 0.012s	sys 0.000s
00:27:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.182426  1921 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Time spent starting tablet: real 0.056s	user 0.040s	sys 0.014s
00:27:57.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.199602  1922 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:57.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.200503  1922 raft_consensus.cc:385] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:27:57.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.200773  1922 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d0f7154f76e840bc92259656e7fd7d0b, State: Initialized, Role: FOLLOWER
00:27:57.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.207086  1922 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:57.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.209852  1923 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:57.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.210738  1923 raft_consensus.cc:385] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:27:57.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.211033  1923 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f84a67fab33464bb93a32fd5b934b8d, State: Initialized, Role: FOLLOWER
00:27:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.212344  1923 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:57.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.215706  1922 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Time spent starting tablet: real 0.062s	user 0.030s	sys 0.003s
00:27:57.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.222038  1923 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Time spent starting tablet: real 0.058s	user 0.040s	sys 0.000s
00:27:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.242612  1928 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:27:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.243032  1928 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:57.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.245635  1928 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:27:57.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:57.256209  1889 tablet.cc:2406] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:27:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:57.259420  1614 tablet.cc:2406] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:27:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.259696  1700 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0f84a67fab33464bb93a32fd5b934b8d" is_pre_election: true
00:27:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.260571  1700 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d0f7154f76e840bc92259656e7fd7d0b in term 0.
00:27:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.260728  1840 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" is_pre_election: true
00:27:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.261624  1840 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d0f7154f76e840bc92259656e7fd7d0b in term 0.
00:27:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.261914  1493 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d0f7154f76e840bc92259656e7fd7d0b; no voters: 
00:27:57.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.262764  1928 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 0 FOLLOWER]: Leader pre-election won for term 1
00:27:57.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.263028  1928 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:27:57.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.263329  1928 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 0 FOLLOWER]: Advancing to term 1
00:27:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.267596  1928 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.269552  1928 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 1 election: Requested vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:27:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.269958  1840 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:27:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.270304  1700 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0f84a67fab33464bb93a32fd5b934b8d"
00:27:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.270470  1840 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 0 FOLLOWER]: Advancing to term 1
00:27:57.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.270820  1700 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 0 FOLLOWER]: Advancing to term 1
00:27:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.275383  1700 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d0f7154f76e840bc92259656e7fd7d0b in term 1.
00:27:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.276258  1493 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d0f7154f76e840bc92259656e7fd7d0b; no voters: 
00:27:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.276891  1928 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 1 FOLLOWER]: Leader election won for term 1
00:27:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.277036  1840 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d0f7154f76e840bc92259656e7fd7d0b in term 1.
00:27:57.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.283536  1928 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 1 LEADER]: Becoming Leader. State: Replica: d0f7154f76e840bc92259656e7fd7d0b, State: Running, Role: LEADER
00:27:57.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.284544  1928 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:57.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.294907  1371 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b reported cstate change: term changed from 0 to 1, leader changed from <none> to d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1). New cstate: current_term: 1 leader_uuid: "d0f7154f76e840bc92259656e7fd7d0b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: HEALTHY } } }
00:27:57.332 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table be871dc169c84c79815b569640956aeb
00:27:57.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:57.338637  1754 tablet.cc:2406] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:27:57.353 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:27:57.370 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299983, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:27:57.372 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=30), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [10ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=null) after lookup
00:27:57.383 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:27:57.387 [DEBUG - kudu-nio-3] (Connection.java:757) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] Successfully connected to peer
00:27:57.443 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xdb93bcea, L:/127.0.0.1:35962 - R:/127.1.37.1:36133] using TOKEN/null
00:27:57.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.457146  1700 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 1 FOLLOWER]: Refusing update from remote peer d0f7154f76e840bc92259656e7fd7d0b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:27:57.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.458709  1928 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:27:57.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.459146  1840 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 1 FOLLOWER]: Refusing update from remote peer d0f7154f76e840bc92259656e7fd7d0b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:27:57.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.460819  1928 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:27:57.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.484656  1937 mvcc.cc:204] Tried to move back new op lower bound from 7305095075648913408 to 7305095074964594688. Current Snapshot: MvccSnapshot[applied={T|T < 7305095075648913408}]
00:27:57.629 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:27:57.630 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.37.1:36133
00:27:57.631 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:27:57.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:57.632084  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 1454
00:27:57.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:27:57.658 [DEBUG - kudu-nio-3] (Connection.java:239) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] handling channelInactive
00:27:57.658 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state READY due to: connection closed
00:27:57.658 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: connection closed
00:27:57.667 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:27:57.667 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:27:57.668 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [3ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@86256690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@4cf4dd2e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3f35494f -> wakeup thread Time-limited test)) after lookup
00:27:57.668 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:27:57.671 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.671 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.671 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.688 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:27:57.688 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:27:57.689 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [3ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [13ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [16ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [16ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@86256690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@4cf4dd2e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@315373895 after retry RPC@1525027720), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3f35494f -> wakeup thread Time-limited test -> (continuation of Deferred@315373895 after retry RPC@1525027720))) after lookup
00:27:57.689 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:27:57.691 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.691 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.692 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.727 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:27:57.727 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:27:57.728 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [3ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [13ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [16ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [16ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [34ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [37ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [37ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [71ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@86256690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@4cf4dd2e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@315373895 after retry RPC@1525027720) -> (continuation of Deferred@348800203 after retry RPC@919455676), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3f35494f -> wakeup thread Time-limited test -> (continuation of Deferred@315373895 after retry RPC@1525027720) -> (continuation of Deferred@348800203 after retry RPC@919455676))) after lookup
00:27:57.728 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:27:57.730 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.730 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.730 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.867 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:27:57.867 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:27:57.868 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=213), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [3ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [13ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [16ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [16ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [34ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [37ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [37ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [71ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [73ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [75ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [75ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [208ms] refreshing cache from master, [208ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [212ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@86256690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@4cf4dd2e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@315373895 after retry RPC@1525027720) -> (continuation of Deferred@348800203 after retry RPC@919455676) -> (continuation of Deferred@1935030807 after retry RPC@408386871), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3f35494f -> wakeup thread Time-limited test -> (continuation of Deferred@315373895 after retry RPC@1525027720) -> (continuation of Deferred@348800203 after retry RPC@919455676) -> (continuation of Deferred@1935030807 after retry RPC@408386871))) after lookup
00:27:57.869 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:27:57.870 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.870 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:57.870 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:58.328 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:27:58.328 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:27:58.329 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=673), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [3ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [13ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [16ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [16ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [34ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [37ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [37ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [71ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [73ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [75ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [75ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [208ms] refreshing cache from master, [208ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [212ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [213ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [215ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [215ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [668ms] refreshing cache from master, [668ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [672ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@86256690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@4cf4dd2e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@315373895 after retry RPC@1525027720) -> (continuation of Deferred@348800203 after retry RPC@919455676) -> (continuation of Deferred@1935030807 after retry RPC@408386871) -> (continuation of Deferred@1759430470 after retry RPC@786756952), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3f35494f -> wakeup thread Time-limited test -> (continuation of Deferred@315373895 after retry RPC@1525027720) -> (continuation of Deferred@348800203 after retry RPC@919455676) -> (continuation of Deferred@1935030807 after retry RPC@408386871) -> (continuation of Deferred@1759430470 after retry RPC@786756952))) after lookup
00:27:58.330 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:27:58.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:58.331 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:58.331 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:27:59.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.079059  1951 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d0f7154f76e840bc92259656e7fd7d0b)
00:27:59.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.079524  1952 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d0f7154f76e840bc92259656e7fd7d0b)
00:27:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.079490  1951 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.079886  1952 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.082036  1952 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:27:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.082023  1951 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:27:59.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:59.094506  1633 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:27:59.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:59.097422  1773 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:27:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.108836  1840 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "0f84a67fab33464bb93a32fd5b934b8d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" is_pre_election: true
00:27:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:59.108870  1773 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:27:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.109464  1840 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f84a67fab33464bb93a32fd5b934b8d in term 1.
00:27:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:59.109843  1633 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:27:59.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.110638  1635 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: d0f7154f76e840bc92259656e7fd7d0b
00:27:59.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.111577  1952 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 1 FOLLOWER]: Leader pre-election won for term 2
00:27:59.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.111937  1952 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 1 FOLLOWER]: Starting leader election (detected failure of leader d0f7154f76e840bc92259656e7fd7d0b)
00:27:59.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.112264  1952 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 1 FOLLOWER]: Advancing to term 2
00:27:59.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.113915  1700 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "0f84a67fab33464bb93a32fd5b934b8d" is_pre_election: true
00:27:59.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.116882  1952 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:59.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.117695  1700 raft_consensus.cc:2393] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d7f44bee9e9544549e3b9aebe7adbdfe in current term 2: Already voted for candidate 0f84a67fab33464bb93a32fd5b934b8d in this term.
00:27:59.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.118348  1952 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 2 election: Requested vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:27:59.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.118978  1840 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "0f84a67fab33464bb93a32fd5b934b8d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:27:59.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.119495  1840 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 1 FOLLOWER]: Advancing to term 2
00:27:59.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.121133  1774 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d7f44bee9e9544549e3b9aebe7adbdfe; no voters: 0f84a67fab33464bb93a32fd5b934b8d, d0f7154f76e840bc92259656e7fd7d0b
00:27:59.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:59.121635  1633 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:27:59.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.124296  1840 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f84a67fab33464bb93a32fd5b934b8d in term 2.
00:27:59.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.124693  1951 raft_consensus.cc:2749] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
00:27:59.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.125130  1635 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: d0f7154f76e840bc92259656e7fd7d0b
00:27:59.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.125829  1952 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 2 FOLLOWER]: Leader election won for term 2
00:27:59.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.127595  1952 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 2 LEADER]: Becoming Leader. State: Replica: 0f84a67fab33464bb93a32fd5b934b8d, State: Running, Role: LEADER
00:27:59.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.128470  1952 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:27:59.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.136828  1371 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d reported cstate change: term changed from 1 to 2, leader changed from d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1) to 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2). New cstate: current_term: 2 leader_uuid: "0f84a67fab33464bb93a32fd5b934b8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: UNKNOWN } } }
00:27:59.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.585559  1840 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 2 FOLLOWER]: Refusing update from remote peer 0f84a67fab33464bb93a32fd5b934b8d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
00:27:59.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:27:59.586761  1952 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
00:27:59.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:27:59.606731  1633 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d -> Peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Couldn't send request to peer d0f7154f76e840bc92259656e7fd7d0b. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:00.127 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:00.128 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:00.129 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=2474), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [3ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: connection closed, [3ms] delaying RPC due to: Network error: connection closed, [7ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [13ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [16ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [16ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [34ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [37ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [37ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [71ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [73ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [75ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [75ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [208ms] refreshing cache from master, [208ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [212ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [213ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [215ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [215ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [668ms] refreshing cache from master, [668ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [672ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [674ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [676ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [676ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2468ms] refreshing cache from master, [2469ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [2472ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@86256690(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@4cf4dd2e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@315373895 after retry RPC@1525027720) -> (continuation of Deferred@348800203 after retry RPC@919455676) -> (continuation of Deferred@1935030807 after retry RPC@408386871) -> (continuation of Deferred@1759430470 after retry RPC@786756952) -> (continuation of Deferred@1633207054 after retry RPC@1354238452), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3f35494f -> wakeup thread Time-limited test -> (continuation of Deferred@315373895 after retry RPC@1525027720) -> (continuation of Deferred@348800203 after retry RPC@919455676) -> (continuation of Deferred@1935030807 after retry RPC@408386871) -> (continuation of Deferred@1759430470 after retry RPC@786756952) -> (continuation of Deferred@1633207054 after retry RPC@1354238452))) after lookup
00:28:00.130 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:00.131 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] Successfully connected to peer
00:28:00.156 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xb7794d38, L:/127.0.0.1:51824 - R:/127.1.37.2:37155] using TOKEN/null
00:28:00.202 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.37.1:36133
00:28:00.204 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:00.205222  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/logs
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.1:36133
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.1
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44645
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.1
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:00.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:00.499265  1987 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:00.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:00.499733  1987 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:00.500183  1987 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:00.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:00.531678  1987 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:00.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:00.532542  1987 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.1
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:00.568689  1987 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.1:36133
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.1
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44645
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.1
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/logs
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:28:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:00.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:00.569980  1987 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:00.571640  1987 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:00.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:00.583467  1993 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:01.986438  1992 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 1987
00:28:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:00.584810  1994 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:02.082190  1992 kernel_stack_watchdog.cc:198] Thread 1987 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 399ms:
00:28:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
00:28:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
00:28:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
00:28:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
00:28:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:02.086349  1995 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1498 milliseconds
00:28:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:02.086563  1987 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.502s	user 0.000s	sys 0.002s
00:28:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:02.087218  1987 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.503s	user 0.000s	sys 0.002s
00:28:02.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:02.089609  1996 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:02.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.089620  1987 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:02.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.090802  1987 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.093250  1987 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:02.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.094647  1987 hybrid_clock.cc:648] HybridClock initialized: now 1783470482094601 us; error 51 us; skew 500 ppm
00:28:02.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.095383  1987 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.101307  1987 webserver.cc:534] Webserver started at http://127.1.37.1:44645/ using document root <none> and password file <none>
00:28:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.102119  1987 fs_manager.cc:362] Metadata directory not provided
00:28:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.102344  1987 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:02.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.109987  1987 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.000s	sys 0.004s
00:28:02.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.114367  2010 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:02.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.115387  1987 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
00:28:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.115718  1987 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0f7154f76e840bc92259656e7fd7d0b"
00:28:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:51 on dist-test-slave-4l7f"
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.117588  1987 fs_report.cc:389] FS layout report
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/data
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:02.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.169019  1987 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:02.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.170519  1987 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:02.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.170997  1987 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:02.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.173620  1987 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.179255  2017 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:28:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.187152  1987 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.187459  1987 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.000s
00:28:02.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.187780  1987 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:02.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.193699  1987 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:02.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.193876  1987 ts_tablet_manager.cc:595] Time spent register tablets: real 0.006s	user 0.005s	sys 0.000s
00:28:02.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.194336  2017 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap starting.
00:28:02.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:02.243899  1633 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d -> Peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Couldn't send request to peer d0f7154f76e840bc92259656e7fd7d0b. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:28:02.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.243947  2017 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Log is configured to *not* fsync() on all Append() calls
00:28:02.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.275954  2017 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:02.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.276938  2017 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap complete.
00:28:02.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.278252  2017 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Time spent bootstrapping tablet: real 0.084s	user 0.038s	sys 0.042s
00:28:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.295322  2017 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.297628  2017 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d0f7154f76e840bc92259656e7fd7d0b, State: Initialized, Role: FOLLOWER
00:28:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.298681  2017 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:02.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.302883  2017 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Time spent starting tablet: real 0.024s	user 0.012s	sys 0.013s
00:28:02.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.351961  1987 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.1:36133
00:28:02.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.352135  2125 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.1:36133 every 8 connection(s)
00:28:02.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.355242  1987 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:02.357 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:02.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.356809  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 1987
00:28:02.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.387871  2126 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:28:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.391227  2126 heartbeater.cc:461] Registering TS with master...
00:28:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.392524  2126 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:28:02.397 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:02.398 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.37.2:37155
00:28:02.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:02.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.398962  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 1613
00:28:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.399667  1371 ts_manager.cc:194] Re-registered known tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:02.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.403446  1371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.1:48335
00:28:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.407143  2127 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:28:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.407472  2127 heartbeater.cc:461] Registering TS with master...
00:28:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.408103  2127 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:28:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.409667  2128 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:28:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.409999  2128 heartbeater.cc:461] Registering TS with master...
00:28:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.410617  2128 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:28:02.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.411495  1292 ts_manager.cc:194] Re-registered known tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:02.414317  1217 ts_manager.cc:194] Re-registered known tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:02.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:02.426 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] handling channelInactive
00:28:02.426 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state READY due to: connection closed
00:28:02.427 [DEBUG - Time-limited test] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:02.428 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.428 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.428 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.447 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:02.447 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:02.447 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=21), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1159970391(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@55c4f8ae -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12194c56 -> wakeup thread Time-limited test)) after lookup
00:28:02.448 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:02.449 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.449 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.449 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.466 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:02.467 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:02.467 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [22ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [23ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1159970391(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@55c4f8ae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12194c56 -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007))) after lookup
00:28:02.468 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:02.469 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.469 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.469 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.507 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:02.508 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:02.508 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=82), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [22ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [23ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [43ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1159970391(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@55c4f8ae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12194c56 -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067))) after lookup
00:28:02.509 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:02.510 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.510 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.511 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.567 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:02.567 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:02.568 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=141), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [22ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [23ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [43ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [83ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [84ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [85ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [141ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1159970391(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@55c4f8ae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067) -> (continuation of Deferred@734922317 after retry RPC@989124281), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12194c56 -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067) -> (continuation of Deferred@734922317 after retry RPC@989124281))) after lookup
00:28:02.569 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:02.570 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.570 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.570 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.807 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:02.807 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:02.808 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=381), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [22ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [23ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [43ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [83ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [84ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [85ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [141ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [142ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [144ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [144ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [377ms] refreshing cache from master, [377ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [381ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1159970391(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@55c4f8ae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067) -> (continuation of Deferred@734922317 after retry RPC@989124281) -> (continuation of Deferred@340100964 after retry RPC@144235083), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12194c56 -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067) -> (continuation of Deferred@734922317 after retry RPC@989124281) -> (continuation of Deferred@340100964 after retry RPC@144235083))) after lookup
00:28:02.809 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:02.810 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.810 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:02.810 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:03.207 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:03.208 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:03.208 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=781), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [22ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [23ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [43ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [83ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [84ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [85ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [141ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [142ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [144ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [144ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [377ms] refreshing cache from master, [377ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [381ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [382ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [384ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [384ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [777ms] refreshing cache from master, [778ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [781ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1159970391(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@55c4f8ae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067) -> (continuation of Deferred@734922317 after retry RPC@989124281) -> (continuation of Deferred@340100964 after retry RPC@144235083) -> (continuation of Deferred@1388603287 after retry RPC@1508042536), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12194c56 -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067) -> (continuation of Deferred@734922317 after retry RPC@989124281) -> (continuation of Deferred@340100964 after retry RPC@144235083) -> (continuation of Deferred@1388603287 after retry RPC@1508042536))) after lookup
00:28:03.209 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:03.210 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:03.210 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:03.210 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.398643  2139 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 0f84a67fab33464bb93a32fd5b934b8d)
00:28:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.399134  2139 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.400629  2139 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:03.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:03.405680  1774 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.407280  2126 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:28:03.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.413149  2080 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "d0f7154f76e840bc92259656e7fd7d0b" is_pre_election: true
00:28:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.413894  2080 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d7f44bee9e9544549e3b9aebe7adbdfe in term 1.
00:28:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.415181  1773 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d0f7154f76e840bc92259656e7fd7d0b, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: 0f84a67fab33464bb93a32fd5b934b8d
00:28:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.416236  2139 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 2 FOLLOWER]: Leader pre-election won for term 3
00:28:03.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.416738  2139 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 2 FOLLOWER]: Starting leader election (detected failure of leader 0f84a67fab33464bb93a32fd5b934b8d)
00:28:03.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.417201  2139 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 2 FOLLOWER]: Advancing to term 3
00:28:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.421900  2139 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.423260  2139 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 3 election: Requested vote from peers 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:03.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.424355  2080 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "d0f7154f76e840bc92259656e7fd7d0b"
00:28:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.424916  2080 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 1 FOLLOWER]: Advancing to term 3
00:28:03.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:03.426949  1774 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:03.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.431490  2080 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d7f44bee9e9544549e3b9aebe7adbdfe in term 3.
00:28:03.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.432284  1773 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d0f7154f76e840bc92259656e7fd7d0b, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: 0f84a67fab33464bb93a32fd5b934b8d
00:28:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.432890  2139 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 3 FOLLOWER]: Leader election won for term 3
00:28:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.433866  2139 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 3 LEADER]: Becoming Leader. State: Replica: d7f44bee9e9544549e3b9aebe7adbdfe, State: Running, Role: LEADER
00:28:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.434685  2139 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.441967  1371 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe reported cstate change: term changed from 2 to 3, leader changed from 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2) to d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3). New cstate: current_term: 3 leader_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: UNKNOWN } } }
00:28:03.449 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:03.449 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:03.450 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=14, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1023), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [22ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [23ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [43ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [83ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [84ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [85ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [141ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [142ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [144ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [144ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [377ms] refreshing cache from master, [377ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [381ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [382ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [384ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [384ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [777ms] refreshing cache from master, [778ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [781ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [782ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [784ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [784ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [1017ms] refreshing cache from master, [1018ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [1022ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1159970391(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@55c4f8ae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067) -> (continuation of Deferred@734922317 after retry RPC@989124281) -> (continuation of Deferred@340100964 after retry RPC@144235083) -> (continuation of Deferred@1388603287 after retry RPC@1508042536) -> (continuation of Deferred@1378558051 after retry RPC@1891214544), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12194c56 -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067) -> (continuation of Deferred@734922317 after retry RPC@989124281) -> (continuation of Deferred@340100964 after retry RPC@144235083) -> (continuation of Deferred@1388603287 after retry RPC@1508042536) -> (continuation of Deferred@1378558051 after retry RPC@1891214544))) after lookup
00:28:03.451 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:03.452 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:03.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:03.452 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:03.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:03.957559  1774 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe -> Peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Couldn't send request to peer 0f84a67fab33464bb93a32fd5b934b8d. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:03.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:03.998944  2080 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 3 FOLLOWER]: Refusing update from remote peer d7f44bee9e9544549e3b9aebe7adbdfe: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
00:28:04.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:04.000305  2157 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [LEADER]: Connected to new peer: Peer: permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
00:28:04.287 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:04.288 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:04.288 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=16, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1861), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [21ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [22ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [23ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [43ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [77ms] refreshing cache from master, [77ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [81ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [83ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [84ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [85ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [137ms] refreshing cache from master, [137ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [141ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [142ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [144ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [144ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [377ms] refreshing cache from master, [377ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [381ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [382ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [384ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [384ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [777ms] refreshing cache from master, [778ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [781ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [782ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [784ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [784ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [1017ms] refreshing cache from master, [1018ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [1022ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [1024ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [1026ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [1026ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [1857ms] refreshing cache from master, [1857ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [1861ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1159970391(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@55c4f8ae -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067) -> (continuation of Deferred@734922317 after retry RPC@989124281) -> (continuation of Deferred@340100964 after retry RPC@144235083) -> (continuation of Deferred@1388603287 after retry RPC@1508042536) -> (continuation of Deferred@1378558051 after retry RPC@1891214544) -> (continuation of Deferred@516833457 after retry RPC@602728604), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@12194c56 -> wakeup thread Time-limited test -> (continuation of Deferred@2028635008 after retry RPC@541662007) -> (continuation of Deferred@451167787 after retry RPC@1914468067) -> (continuation of Deferred@734922317 after retry RPC@989124281) -> (continuation of Deferred@340100964 after retry RPC@144235083) -> (continuation of Deferred@1388603287 after retry RPC@1508042536) -> (continuation of Deferred@1378558051 after retry RPC@1891214544) -> (continuation of Deferred@516833457 after retry RPC@602728604))) after lookup
00:28:04.289 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:04.290 [DEBUG - kudu-nio-2] (Connection.java:757) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] Successfully connected to peer
00:28:04.310 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xaf911b20, L:/127.0.0.1:56214 - R:/127.1.37.3:36659] using TOKEN/null
00:28:04.368 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.37.2:37155
00:28:04.369 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:04.371564  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/logs
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.2:37155
00:28:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.2
00:28:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33733
00:28:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.2
00:28:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:04.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:04.553213  1774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111) [suppressed 3 similar messages]
00:28:04.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:04.674238  2171 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:04.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:04.674768  2171 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:04.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:04.675416  2171 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:04.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:04.707157  2171 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:04.708123  2171 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.2
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:04.742877  2171 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.2:37155
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.2
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33733
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.2
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/logs
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:04.744621  2171 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:04.746304  2171 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:04.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:04.758126  2179 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.160727  2178 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 2171
00:28:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.176124  2171 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.418s	user 0.543s	sys 0.866s
00:28:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.176895  2171 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.419s	user 0.543s	sys 0.866s
00:28:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:04.758589  2180 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:06.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.177886  2181 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1418 milliseconds
00:28:06.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.178344  2171 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:06.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.178414  2182 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:06.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.181516  2171 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.183583  2171 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:06.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.185056  2171 hybrid_clock.cc:648] HybridClock initialized: now 1783470486185018 us; error 37 us; skew 500 ppm
00:28:06.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.185829  2171 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:06.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.191452  2171 webserver.cc:534] Webserver started at http://127.1.37.2:33733/ using document root <none> and password file <none>
00:28:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.192281  2171 fs_manager.cc:362] Metadata directory not provided
00:28:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.192528  2171 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.199854  2171 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.001s
00:28:06.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.204197  2193 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:06.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.205221  2171 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
00:28:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.205571  2171 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f84a67fab33464bb93a32fd5b934b8d"
00:28:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:53 on dist-test-slave-4l7f"
00:28:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.207435  2171 fs_report.cc:389] FS layout report
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/data
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:06.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.253571  2171 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:06.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.255008  2171 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:06.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.255402  2171 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:06.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.257916  2171 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:06.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.263325  2200 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:28:06.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.271179  2171 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:06.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.271402  2171 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.001s	sys 0.001s
00:28:06.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.271749  2171 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.276746  2171 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.276930  2171 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.003s
00:28:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.277267  2200 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Bootstrap starting.
00:28:06.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.333166  2200 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Log is configured to *not* fsync() on all Append() calls
00:28:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.379878  2200 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:06.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.380872  2200 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Bootstrap complete.
00:28:06.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.382541  2200 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Time spent bootstrapping tablet: real 0.106s	user 0.061s	sys 0.039s
00:28:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.403009  2200 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.406340  2200 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f84a67fab33464bb93a32fd5b934b8d, State: Initialized, Role: FOLLOWER
00:28:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.407433  2200 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:06.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.411738  2200 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Time spent starting tablet: real 0.029s	user 0.023s	sys 0.006s
00:28:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.458878  2171 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.2:37155
00:28:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.459110  2308 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.2:37155 every 8 connection(s)
00:28:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.461234  2171 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:28:06.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.465050  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 2171
00:28:06.466 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:06.501 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:06.501 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.37.3:36659
00:28:06.502 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.502801  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 1755
00:28:06.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.516459  2311 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:28:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.516939  2311 heartbeater.cc:461] Registering TS with master...
00:28:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.517168  2309 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:28:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.517499  2309 heartbeater.cc:461] Registering TS with master...
00:28:06.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.518265  2311 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:28:06.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.518620  2309 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:28:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.522322  1372 ts_manager.cc:194] Re-registered known tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.522957  1216 ts_manager.cc:194] Re-registered known tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:06.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.525475  1372 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.2:44573
00:28:06.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.526989  2310 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:28:06.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.527347  2310 heartbeater.cc:461] Registering TS with master...
00:28:06.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.528083  2310 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:28:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:06.530934  1292 ts_manager.cc:194] Re-registered known tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:06.534 [DEBUG - kudu-nio-2] (Connection.java:239) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] handling channelInactive
00:28:06.534 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state READY due to: connection closed
00:28:06.535 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:06.536 [DEBUG - Time-limited test] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:06.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:06.534770  2321 negotiation.cc:336] Failed RPC negotiation. Trace:
00:28:06.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:06.490429 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.1.37.3:42871 (local address 127.1.37.2:37155)
00:28:06.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:06.505033 (+ 14604us) server_negotiation.cc:207] Beginning negotiation
00:28:06.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:06.505067 (+    34us) server_negotiation.cc:400] Waiting for connection header
00:28:06.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:06.533410 (+ 28343us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.37.3:42871: BlockingRecv error: recv got EOF from 127.1.37.3:42871 (error 108)
00:28:06.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":13886,"thread_start_us":13500,"threads_started":1}
00:28:06.537 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.537 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.537 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.548 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:06.548 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:06.548 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1451743394(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@126b72e6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@558429dc -> wakeup thread Time-limited test)) after lookup
00:28:06.549 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:06.550 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.550 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.550 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.566 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:06.567 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:06.567 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [14ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [15ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [15ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1451743394(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@126b72e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@558429dc -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613))) after lookup
00:28:06.568 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:06.569 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.569 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.569 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.607 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:06.608 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:06.608 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [14ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [15ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [15ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [32ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [34ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [34ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [68ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [72ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1451743394(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@126b72e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613) -> (continuation of Deferred@1402299290 after retry RPC@68941877), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@558429dc -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613) -> (continuation of Deferred@1402299290 after retry RPC@68941877))) after lookup
00:28:06.609 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:06.610 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.610 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.610 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.748 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:06.748 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:06.749 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=213), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [14ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [15ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [15ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [32ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [34ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [34ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [68ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [72ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [73ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [75ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [75ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [208ms] refreshing cache from master, [208ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [213ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1451743394(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@126b72e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613) -> (continuation of Deferred@1402299290 after retry RPC@68941877) -> (continuation of Deferred@1110710106 after retry RPC@431330681), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@558429dc -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613) -> (continuation of Deferred@1402299290 after retry RPC@68941877) -> (continuation of Deferred@1110710106 after retry RPC@431330681))) after lookup
00:28:06.749 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:06.751 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.751 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:06.751 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:07.267 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:07.267 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:07.268 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=732), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [14ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [15ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [15ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [32ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [34ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [34ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [68ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [72ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [73ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [75ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [75ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [208ms] refreshing cache from master, [208ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [213ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [214ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [216ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [216ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [728ms] refreshing cache from master, [728ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [732ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1451743394(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@126b72e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613) -> (continuation of Deferred@1402299290 after retry RPC@68941877) -> (continuation of Deferred@1110710106 after retry RPC@431330681) -> (continuation of Deferred@189586706 after retry RPC@1055362911), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@558429dc -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613) -> (continuation of Deferred@1402299290 after retry RPC@68941877) -> (continuation of Deferred@1110710106 after retry RPC@431330681) -> (continuation of Deferred@189586706 after retry RPC@1055362911))) after lookup
00:28:07.268 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:07.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:07.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:07.269 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.528887  2309 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:28:07.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.908293  2323 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 3 FOLLOWER]: Starting pre-election (detected failure of leader d7f44bee9e9544549e3b9aebe7adbdfe)
00:28:07.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.908730  2323 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:07.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.910913  2323 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:07.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:07.913659  2013 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111)
00:28:07.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:07.917724  2013 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111)
00:28:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.923043  2263 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "0f84a67fab33464bb93a32fd5b934b8d" is_pre_election: true
00:28:07.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.923763  2263 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d0f7154f76e840bc92259656e7fd7d0b in term 2.
00:28:07.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.925303  2012 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d0f7154f76e840bc92259656e7fd7d0b; no voters: d7f44bee9e9544549e3b9aebe7adbdfe
00:28:07.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.926083  2323 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 3 FOLLOWER]: Leader pre-election won for term 4
00:28:07.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.926319  2323 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 3 FOLLOWER]: Starting leader election (detected failure of leader d7f44bee9e9544549e3b9aebe7adbdfe)
00:28:07.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.926558  2323 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 3 FOLLOWER]: Advancing to term 4
00:28:07.928 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:07.928 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:07.929 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1393), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [14ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [15ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [15ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [32ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [34ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [34ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [68ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [72ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [73ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [75ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [75ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [208ms] refreshing cache from master, [208ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [213ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [214ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [216ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [216ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [728ms] refreshing cache from master, [728ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [732ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [733ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [734ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [734ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1388ms] refreshing cache from master, [1388ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [1393ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1451743394(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@126b72e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613) -> (continuation of Deferred@1402299290 after retry RPC@68941877) -> (continuation of Deferred@1110710106 after retry RPC@431330681) -> (continuation of Deferred@189586706 after retry RPC@1055362911) -> (continuation of Deferred@626908476 after retry RPC@287486809), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@558429dc -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613) -> (continuation of Deferred@1402299290 after retry RPC@68941877) -> (continuation of Deferred@1110710106 after retry RPC@431330681) -> (continuation of Deferred@189586706 after retry RPC@1055362911) -> (continuation of Deferred@626908476 after retry RPC@287486809))) after lookup
00:28:07.929 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:07.930 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:07.931 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:07.931 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.931416  2323 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:07.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.932883  2323 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 4 election: Requested vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.933959  2263 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "0f84a67fab33464bb93a32fd5b934b8d"
00:28:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.934471  2263 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 2 FOLLOWER]: Advancing to term 4
00:28:07.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:07.936713  2013 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111)
00:28:07.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.941396  2263 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d0f7154f76e840bc92259656e7fd7d0b in term 4.
00:28:07.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.942117  2012 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d0f7154f76e840bc92259656e7fd7d0b; no voters: d7f44bee9e9544549e3b9aebe7adbdfe
00:28:07.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.942682  2323 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 4 FOLLOWER]: Leader election won for term 4
00:28:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.944207  2323 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 4 LEADER]: Becoming Leader. State: Replica: d0f7154f76e840bc92259656e7fd7d0b, State: Running, Role: LEADER
00:28:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.944976  2323 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:07.954010  1372 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b reported cstate change: term changed from 3 to 4, leader changed from d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3) to d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1). New cstate: current_term: 4 leader_uuid: "d0f7154f76e840bc92259656e7fd7d0b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: HEALTHY } } }
00:28:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:08.417562  2013 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b -> Peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Couldn't send request to peer d7f44bee9e9544549e3b9aebe7adbdfe. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:08.448968  2263 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 4 FOLLOWER]: Refusing update from remote peer d0f7154f76e840bc92259656e7fd7d0b: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
00:28:08.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:08.450497  2323 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
00:28:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:10.805154  2013 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b -> Peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Couldn't send request to peer d7f44bee9e9544549e3b9aebe7adbdfe. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:28:11.747 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:11.747 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:11.748 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=5212), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [12ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [14ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [15ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [15ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [31ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [32ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [34ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [34ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [68ms] refreshing cache from master, [69ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [72ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [73ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [75ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [75ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [208ms] refreshing cache from master, [208ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [213ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [214ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [216ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [216ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [728ms] refreshing cache from master, [728ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [732ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [733ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [734ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [734ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1388ms] refreshing cache from master, [1388ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [1393ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [1394ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [1396ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1396ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [5208ms] refreshing cache from master, [5208ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [5211ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1451743394(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@126b72e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613) -> (continuation of Deferred@1402299290 after retry RPC@68941877) -> (continuation of Deferred@1110710106 after retry RPC@431330681) -> (continuation of Deferred@189586706 after retry RPC@1055362911) -> (continuation of Deferred@626908476 after retry RPC@287486809) -> (continuation of Deferred@655485216 after retry RPC@590163680), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@558429dc -> wakeup thread Time-limited test -> (continuation of Deferred@1214479004 after retry RPC@1012756613) -> (continuation of Deferred@1402299290 after retry RPC@68941877) -> (continuation of Deferred@1110710106 after retry RPC@431330681) -> (continuation of Deferred@189586706 after retry RPC@1055362911) -> (continuation of Deferred@626908476 after retry RPC@287486809) -> (continuation of Deferred@655485216 after retry RPC@590163680))) after lookup
00:28:11.749 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:11.750 [DEBUG - kudu-nio-2] (Connection.java:757) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] Successfully connected to peer
00:28:11.775 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x1433a68d, L:/127.0.0.1:39248 - R:/127.1.37.1:36133] using TOKEN/null
00:28:11.824 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.37.3:36659
00:28:11.824 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:11.825980  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/logs
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.3:36659
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.3
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44545
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.3
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:12.119841  2365 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:12.120296  2365 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:12.120790  2365 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:12.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:12.153455  2365 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:12.154259  2365 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.3
00:28:12.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:12.189479  2365 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:12.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.3:36659
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.3
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44545
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.3
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/logs
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:12.190708  2365 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:12.192236  2365 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:12.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:12.203836  2371 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:13.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.337098  2013 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111) [suppressed 11 similar messages]
00:28:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.360157  2013 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b -> Peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Couldn't send request to peer d7f44bee9e9544549e3b9aebe7adbdfe. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
00:28:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.606808  2370 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 2365
00:28:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.634893  2370 kernel_stack_watchdog.cc:198] Thread 2365 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 398ms:
00:28:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
00:28:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
00:28:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
00:28:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
00:28:13.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:12.204784  2372 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:13.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.635752  2365 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.430s	user 0.001s	sys 0.000s
00:28:13.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.636096  2365 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.431s	user 0.001s	sys 0.000s
00:28:13.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.637648  2374 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:13.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:13.639923  2373 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1429 milliseconds
00:28:13.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.639945  2365 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:13.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.641256  2365 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:13.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.643396  2365 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.644881  2365 hybrid_clock.cc:648] HybridClock initialized: now 1783470493644794 us; error 89 us; skew 500 ppm
00:28:13.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.645686  2365 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.651495  2365 webserver.cc:534] Webserver started at http://127.1.37.3:44545/ using document root <none> and password file <none>
00:28:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.652331  2365 fs_manager.cc:362] Metadata directory not provided
00:28:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.652565  2365 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:13.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.659848  2365 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.004s
00:28:13.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.664433  2387 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:13.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.665585  2365 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.003s
00:28:13.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.665913  2365 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:13.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:28:13.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:55 on dist-test-slave-4l7f"
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.667747  2365 fs_report.cc:389] FS layout report
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/data
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:13.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.716508  2365 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:13.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.717989  2365 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:13.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.718454  2365 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:13.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.721357  2365 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.726796  2394 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:28:13.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.734619  2365 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:13.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.734849  2365 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.002s	sys 0.000s
00:28:13.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.735170  2365 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:13.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.739722  2365 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.739912  2365 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
00:28:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.740218  2394 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Bootstrap starting.
00:28:13.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.803077  2394 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Log is configured to *not* fsync() on all Append() calls
00:28:13.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.848094  2394 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:13.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.848965  2394 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Bootstrap complete.
00:28:13.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.850214  2394 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Time spent bootstrapping tablet: real 0.110s	user 0.077s	sys 0.029s
00:28:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.868978  2394 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.871183  2394 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: d7f44bee9e9544549e3b9aebe7adbdfe, State: Initialized, Role: FOLLOWER
00:28:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.872056  2394 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.876139  2394 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Time spent starting tablet: real 0.026s	user 0.024s	sys 0.000s
00:28:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.900449  2365 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.3:36659
00:28:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.900744  2504 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.3:36659 every 8 connection(s)
00:28:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.902784  2365 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:28:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.909062  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 2365
00:28:13.910 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:13.938 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:13.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.939962  2505 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:28:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.940987  2505 heartbeater.cc:461] Registering TS with master...
00:28:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.942059  2505 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:28:13.947 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.37.1:36133
00:28:13.948 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:13.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.947619  1372 ts_manager.cc:194] Re-registered known tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:28:13.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.948611  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 1987
00:28:13.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.950327  1372 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.3:49101
00:28:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.959123  2506 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:28:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.959439  2506 heartbeater.cc:461] Registering TS with master...
00:28:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.960086  2506 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:28:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.960318  2507 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:28:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.960681  2507 heartbeater.cc:461] Registering TS with master...
00:28:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.961259  2507 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:28:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.963596  1292 ts_manager.cc:194] Re-registered known tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:28:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:13.964565  1216 ts_manager.cc:194] Re-registered known tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:28:13.977 [DEBUG - kudu-nio-2] (Connection.java:239) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] handling channelInactive
00:28:13.977 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state READY due to: connection closed
00:28:13.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:13.978 [DEBUG - Time-limited test] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:13.980 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:13.980 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:13.980 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:13.987 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:13.987 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:13.987 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1695777808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@2f83fc7c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@417b677 -> wakeup thread Time-limited test)) after lookup
00:28:13.988 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:13.989 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:13.989 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:13.989 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.006 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:14.007 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:14.007 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [9ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [28ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1695777808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@2f83fc7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@417b677 -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195))) after lookup
00:28:14.008 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:14.009 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.009 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.009 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.047 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:14.047 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:14.048 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=69), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [9ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [28ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [29ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [31ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [31ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1695777808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@2f83fc7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195) -> (continuation of Deferred@1517512967 after retry RPC@1548606789), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@417b677 -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195) -> (continuation of Deferred@1517512967 after retry RPC@1548606789))) after lookup
00:28:14.048 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:14.049 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.050 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.050 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.066 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:14.066 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:14.067 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=88), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [9ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [28ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [29ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [31ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [31ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [70ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [72ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [72ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [88ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1695777808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@2f83fc7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195) -> (continuation of Deferred@1517512967 after retry RPC@1548606789) -> (continuation of Deferred@1566330638 after retry RPC@1875445269), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@417b677 -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195) -> (continuation of Deferred@1517512967 after retry RPC@1548606789) -> (continuation of Deferred@1566330638 after retry RPC@1875445269))) after lookup
00:28:14.067 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:14.068 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.068 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.068 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.427 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:14.427 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:14.428 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=449), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [9ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [28ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [29ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [31ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [31ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [70ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [72ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [72ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [88ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [89ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [90ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [90ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [445ms] refreshing cache from master, [445ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [449ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1695777808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@2f83fc7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195) -> (continuation of Deferred@1517512967 after retry RPC@1548606789) -> (continuation of Deferred@1566330638 after retry RPC@1875445269) -> (continuation of Deferred@1257331881 after retry RPC@1840190028), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@417b677 -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195) -> (continuation of Deferred@1517512967 after retry RPC@1548606789) -> (continuation of Deferred@1566330638 after retry RPC@1875445269) -> (continuation of Deferred@1257331881 after retry RPC@1840190028))) after lookup
00:28:14.428 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:14.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.430 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.928 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:14.929 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:14.929 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=951), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [9ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [28ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [29ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [31ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [31ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [70ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [72ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [72ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [88ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [89ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [90ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [90ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [445ms] refreshing cache from master, [445ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [449ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [450ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [452ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [452ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [945ms] refreshing cache from master, [946ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [950ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1695777808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@2f83fc7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195) -> (continuation of Deferred@1517512967 after retry RPC@1548606789) -> (continuation of Deferred@1566330638 after retry RPC@1875445269) -> (continuation of Deferred@1257331881 after retry RPC@1840190028) -> (continuation of Deferred@951369171 after retry RPC@2119928268), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@417b677 -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195) -> (continuation of Deferred@1517512967 after retry RPC@1548606789) -> (continuation of Deferred@1566330638 after retry RPC@1875445269) -> (continuation of Deferred@1257331881 after retry RPC@1840190028) -> (continuation of Deferred@951369171 after retry RPC@2119928268))) after lookup
00:28:14.930 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:14.931 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.931 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.931 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:14.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:14.954970  2505 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:28:15.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.336957  2518 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d0f7154f76e840bc92259656e7fd7d0b)
00:28:15.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.337831  2518 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:15.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.341449  2518 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:15.347872  2194 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:28:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:15.351723  2194 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:28:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.355955  2458 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "0f84a67fab33464bb93a32fd5b934b8d" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" is_pre_election: true
00:28:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.356643  2458 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f84a67fab33464bb93a32fd5b934b8d in term 3.
00:28:15.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.357923  2196 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: d0f7154f76e840bc92259656e7fd7d0b
00:28:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.358768  2518 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 4 FOLLOWER]: Leader pre-election won for term 5
00:28:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.359040  2518 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 4 FOLLOWER]: Starting leader election (detected failure of leader d0f7154f76e840bc92259656e7fd7d0b)
00:28:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.359338  2518 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 4 FOLLOWER]: Advancing to term 5
00:28:15.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.364054  2518 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.365562  2518 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 5 election: Requested vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.366451  2458 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "0f84a67fab33464bb93a32fd5b934b8d" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:28:15.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.366962  2458 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 3 FOLLOWER]: Advancing to term 5
00:28:15.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:15.371748  2194 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:28:15.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.374855  2458 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f84a67fab33464bb93a32fd5b934b8d in term 5.
00:28:15.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.375612  2196 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: d0f7154f76e840bc92259656e7fd7d0b
00:28:15.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.376152  2518 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 5 FOLLOWER]: Leader election won for term 5
00:28:15.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.377146  2518 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 5 LEADER]: Becoming Leader. State: Replica: 0f84a67fab33464bb93a32fd5b934b8d, State: Running, Role: LEADER
00:28:15.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.378154  2518 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:15.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.387176  1372 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d reported cstate change: term changed from 4 to 5, leader changed from d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1) to 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2). New cstate: current_term: 5 leader_uuid: "0f84a67fab33464bb93a32fd5b934b8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: UNKNOWN } } }
00:28:15.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.893940  2458 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 5 FOLLOWER]: Refusing update from remote peer 0f84a67fab33464bb93a32fd5b934b8d: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
00:28:15.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.895324  2535 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
00:28:15.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:15.913934  2537 mvcc.cc:204] Tried to move back new op lower bound from 7305095134331105280 to 7305095118630846464. Current Snapshot: MvccSnapshot[applied={T|T < 7305095134331105280}]
00:28:15.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:15.925530  2194 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d -> Peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Couldn't send request to peer d0f7154f76e840bc92259656e7fd7d0b. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:17.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:17.100803  1403 debug-util.cc:398] Leaking SignalData structure 0x7b080004fa20 after lost signal to thread 1334
00:28:17.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:17.101684  1403 debug-util.cc:398] Leaking SignalData structure 0x7b080009b260 after lost signal to thread 1406
00:28:18.308 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:18.308 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:18.309 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=4330), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [8ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [9ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [25ms] refreshing cache from master, [25ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [28ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [29ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [31ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [31ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [65ms] refreshing cache from master, [66ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [69ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [70ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [72ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [72ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [85ms] refreshing cache from master, [85ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [88ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [89ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [90ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [90ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [445ms] refreshing cache from master, [445ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [449ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [450ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [452ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [452ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [945ms] refreshing cache from master, [946ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [950ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [951ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [953ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [953ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [4325ms] refreshing cache from master, [4325ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [4330ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1695777808(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@2f83fc7c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195) -> (continuation of Deferred@1517512967 after retry RPC@1548606789) -> (continuation of Deferred@1566330638 after retry RPC@1875445269) -> (continuation of Deferred@1257331881 after retry RPC@1840190028) -> (continuation of Deferred@951369171 after retry RPC@2119928268) -> (continuation of Deferred@880022139 after retry RPC@381941968), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@417b677 -> wakeup thread Time-limited test -> (continuation of Deferred@425293529 after retry RPC@549773195) -> (continuation of Deferred@1517512967 after retry RPC@1548606789) -> (continuation of Deferred@1566330638 after retry RPC@1875445269) -> (continuation of Deferred@1257331881 after retry RPC@1840190028) -> (continuation of Deferred@951369171 after retry RPC@2119928268) -> (continuation of Deferred@880022139 after retry RPC@381941968))) after lookup
00:28:18.309 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:18.310 [DEBUG - kudu-nio-2] (Connection.java:757) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] Successfully connected to peer
00:28:18.330 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x9d679383, L:/127.0.0.1:58506 - R:/127.1.37.2:37155] using TOKEN/null
00:28:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.391265  2194 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d -> Peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Couldn't send request to peer d0f7154f76e840bc92259656e7fd7d0b. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:28:18.410 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.37.1:36133
00:28:18.411 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.412092  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/logs
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.1:36133
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.1
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44645
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.1
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:18.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.741518  2559 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:18.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.742058  2559 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:18.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.742645  2559 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.774840  2559 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:18.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.775758  2559 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.1
00:28:18.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.811280  2559 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:18.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:18.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:18.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.1:36133
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.1
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44645
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.1
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/logs
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:18.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:28:18.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:28:18.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:18.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.812958  2559 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:18.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:18.814565  2559 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.826084  2565 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:18.826704  2566 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:20.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.013010  2567 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1182 milliseconds
00:28:20.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.013703  2559 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.187s	user 0.401s	sys 0.768s
00:28:20.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.013972  2559 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.188s	user 0.401s	sys 0.768s
00:28:20.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:20.014317  2568 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:20.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.014382  2559 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:20.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.015566  2559 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:20.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.020910  2559 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.022256  2559 hybrid_clock.cc:648] HybridClock initialized: now 1783470500022206 us; error 52 us; skew 500 ppm
00:28:20.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.023080  2559 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:20.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.029937  2559 webserver.cc:534] Webserver started at http://127.1.37.1:44645/ using document root <none> and password file <none>
00:28:20.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.030864  2559 fs_manager.cc:362] Metadata directory not provided
00:28:20.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.031096  2559 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.039088  2559 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
00:28:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.044211  2578 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.045349  2559 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
00:28:20.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.045678  2559 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:20.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0f7154f76e840bc92259656e7fd7d0b"
00:28:20.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:51 on dist-test-slave-4l7f"
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.047817  2559 fs_report.cc:389] FS layout report
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/data
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:20.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.107116  2559 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:20.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.108659  2559 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:20.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.109150  2559 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:20.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.111663  2559 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:20.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.117245  2585 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:28:20.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.124958  2559 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:20.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.125180  2559 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.000s	sys 0.002s
00:28:20.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.125510  2559 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.129918  2559 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.130101  2559 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
00:28:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.130565  2585 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap starting.
00:28:20.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.184564  2585 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Log is configured to *not* fsync() on all Append() calls
00:28:20.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.255643  2585 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:20.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.256755  2585 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap complete.
00:28:20.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.258512  2585 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Time spent bootstrapping tablet: real 0.128s	user 0.090s	sys 0.034s
00:28:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.283483  2585 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.286834  2585 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: d0f7154f76e840bc92259656e7fd7d0b, State: Initialized, Role: FOLLOWER
00:28:20.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.288185  2559 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.1:36133
00:28:20.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.288357  2693 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.1:36133 every 8 connection(s)
00:28:20.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.288040  2585 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:20.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.291581  2559 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:20.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.291749  2585 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Time spent starting tablet: real 0.033s	user 0.033s	sys 0.001s
00:28:20.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.294384  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 2559
00:28:20.295 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:20.343 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:20.343 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.37.2:37155
00:28:20.344 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:20.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.344362  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 2171
00:28:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.359974  2696 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:28:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.360507  2696 heartbeater.cc:461] Registering TS with master...
00:28:20.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.361781  2696 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:28:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.364573  2697 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:28:20.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.364921  2697 heartbeater.cc:461] Registering TS with master...
00:28:20.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.365824  2697 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:28:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.369488  1292 ts_manager.cc:194] Re-registered known tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.370379  1216 ts_manager.cc:194] Re-registered known tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:20.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.374693  2695 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:28:20.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.375028  2695 heartbeater.cc:461] Registering TS with master...
00:28:20.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.375882  2695 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:28:20.380 [DEBUG - kudu-nio-2] (Connection.java:239) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] handling channelInactive
00:28:20.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.378875  1371 ts_manager.cc:194] Re-registered known tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:20.380 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:20.380 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state READY due to: connection closed
00:28:20.381 [DEBUG - Time-limited test] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:20.383 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.383 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.383 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:20.383968  1371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.1:46509
00:28:20.407 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:20.407 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:20.407 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=27), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [3ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [3ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@807817948(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@392a2970 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26dd9924 -> wakeup thread Time-limited test)) after lookup
00:28:20.408 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:20.409 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.409 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.410 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.427 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:20.427 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:20.427 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=46), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [3ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [3ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [28ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [29ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [30ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@807817948(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@392a2970 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26dd9924 -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851))) after lookup
00:28:20.428 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:20.429 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.429 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.429 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.446 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:20.446 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:20.446 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=66), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [3ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [3ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [28ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [29ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [30ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [47ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [49ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [49ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [66ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@807817948(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@392a2970 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851) -> (continuation of Deferred@1147854419 after retry RPC@619300454), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26dd9924 -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851) -> (continuation of Deferred@1147854419 after retry RPC@619300454))) after lookup
00:28:20.447 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:20.448 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.448 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.448 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.546 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:20.547 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:20.547 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=166), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [3ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [3ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [28ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [29ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [30ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [47ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [49ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [49ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [66ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [67ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [68ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [68ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [166ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@807817948(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@392a2970 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851) -> (continuation of Deferred@1147854419 after retry RPC@619300454) -> (continuation of Deferred@2051745219 after retry RPC@694821759), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26dd9924 -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851) -> (continuation of Deferred@1147854419 after retry RPC@619300454) -> (continuation of Deferred@2051745219 after retry RPC@694821759))) after lookup
00:28:20.548 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:20.549 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.549 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.549 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.787 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:20.788 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:20.788 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=407), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [3ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [3ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [28ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [29ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [30ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [47ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [49ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [49ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [66ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [67ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [68ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [68ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [166ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [167ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [169ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [169ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [403ms] refreshing cache from master, [404ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [407ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@807817948(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@392a2970 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851) -> (continuation of Deferred@1147854419 after retry RPC@619300454) -> (continuation of Deferred@2051745219 after retry RPC@694821759) -> (continuation of Deferred@1629625988 after retry RPC@422295556), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26dd9924 -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851) -> (continuation of Deferred@1147854419 after retry RPC@619300454) -> (continuation of Deferred@2051745219 after retry RPC@694821759) -> (continuation of Deferred@1629625988 after retry RPC@422295556))) after lookup
00:28:20.789 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:20.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.790 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.867 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:20.867 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:20.868 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=487), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [3ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [3ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [28ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [29ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [30ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [47ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [49ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [49ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [66ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [67ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [68ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [68ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [166ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [167ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [169ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [169ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [403ms] refreshing cache from master, [404ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [407ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [408ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [410ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [410ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [483ms] refreshing cache from master, [483ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [487ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@807817948(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@392a2970 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851) -> (continuation of Deferred@1147854419 after retry RPC@619300454) -> (continuation of Deferred@2051745219 after retry RPC@694821759) -> (continuation of Deferred@1629625988 after retry RPC@422295556) -> (continuation of Deferred@1664273912 after retry RPC@211366344), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26dd9924 -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851) -> (continuation of Deferred@1147854419 after retry RPC@619300454) -> (continuation of Deferred@2051745219 after retry RPC@694821759) -> (continuation of Deferred@1629625988 after retry RPC@422295556) -> (continuation of Deferred@1664273912 after retry RPC@211366344))) after lookup
00:28:20.868 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:20.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:20.869 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:21.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.387365  2695 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:28:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.689747  2706 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.690167  2706 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:21.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.692276  2706 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:21.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:21.696823  2580 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:21.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:21.700917  2580 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:21.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.705101  2458 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" is_pre_election: true
00:28:21.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.706436  2581 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d0f7154f76e840bc92259656e7fd7d0b; no voters: 0f84a67fab33464bb93a32fd5b934b8d, d7f44bee9e9544549e3b9aebe7adbdfe
00:28:21.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.707118  2706 raft_consensus.cc:2749] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
00:28:21.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.753075  2711 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 0f84a67fab33464bb93a32fd5b934b8d)
00:28:21.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.753461  2711 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:21.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.756060  2711 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:21.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:21.759425  2389 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:21.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:21.763432  2389 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:21.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.767613  2648 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "d0f7154f76e840bc92259656e7fd7d0b" is_pre_election: true
00:28:21.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.768277  2648 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d7f44bee9e9544549e3b9aebe7adbdfe in term 4.
00:28:21.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.769575  2388 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d0f7154f76e840bc92259656e7fd7d0b, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: 0f84a67fab33464bb93a32fd5b934b8d
00:28:21.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.770160  2711 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 5 FOLLOWER]: Leader pre-election won for term 6
00:28:21.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.770391  2711 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 5 FOLLOWER]: Starting leader election (detected failure of leader 0f84a67fab33464bb93a32fd5b934b8d)
00:28:21.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.770607  2711 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 5 FOLLOWER]: Advancing to term 6
00:28:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.774820  2711 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.776183  2711 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 6 election: Requested vote from peers 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.777150  2648 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "d0f7154f76e840bc92259656e7fd7d0b"
00:28:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.777623  2648 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 4 FOLLOWER]: Advancing to term 6
00:28:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:21.779114  2389 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:21.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.783605  2648 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d7f44bee9e9544549e3b9aebe7adbdfe in term 6.
00:28:21.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.784371  2388 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d0f7154f76e840bc92259656e7fd7d0b, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: 0f84a67fab33464bb93a32fd5b934b8d
00:28:21.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.784929  2711 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 6 FOLLOWER]: Leader election won for term 6
00:28:21.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.786362  2711 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 6 LEADER]: Becoming Leader. State: Replica: d7f44bee9e9544549e3b9aebe7adbdfe, State: Running, Role: LEADER
00:28:21.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.787173  2711 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:21.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:21.795856  1371 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe reported cstate change: term changed from 5 to 6, leader changed from 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2) to d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3). New cstate: current_term: 6 leader_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: UNKNOWN } } }
00:28:22.207 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:22.207 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:22.208 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=14, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=1827), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [3ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [3ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [26ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [28ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [29ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [30ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [46ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [47ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [49ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [49ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [63ms] refreshing cache from master, [63ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [66ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [67ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [68ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [68ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [163ms] refreshing cache from master, [163ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [166ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [167ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [169ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [169ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [403ms] refreshing cache from master, [404ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [407ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [408ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [410ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [410ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [483ms] refreshing cache from master, [483ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [487ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [488ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [489ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [489ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [1823ms] refreshing cache from master, [1823ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [1827ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@807817948(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@392a2970 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851) -> (continuation of Deferred@1147854419 after retry RPC@619300454) -> (continuation of Deferred@2051745219 after retry RPC@694821759) -> (continuation of Deferred@1629625988 after retry RPC@422295556) -> (continuation of Deferred@1664273912 after retry RPC@211366344) -> (continuation of Deferred@1178207446 after retry RPC@65563304), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@26dd9924 -> wakeup thread Time-limited test -> (continuation of Deferred@1138990634 after retry RPC@155003851) -> (continuation of Deferred@1147854419 after retry RPC@619300454) -> (continuation of Deferred@2051745219 after retry RPC@694821759) -> (continuation of Deferred@1629625988 after retry RPC@422295556) -> (continuation of Deferred@1664273912 after retry RPC@211366344) -> (continuation of Deferred@1178207446 after retry RPC@65563304))) after lookup
00:28:22.208 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:22.209 [DEBUG - kudu-nio-2] (Connection.java:757) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] Successfully connected to peer
00:28:22.227 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x1e8ac1cd, L:/127.0.0.1:40960 - R:/127.1.37.3:36659] using TOKEN/null
00:28:22.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.238104  2648 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 6 FOLLOWER]: Refusing update from remote peer d7f44bee9e9544549e3b9aebe7adbdfe: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 22. (index mismatch)
00:28:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.239675  2716 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [LEADER]: Connected to new peer: Peer: permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.001s
00:28:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.240957  2389 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe -> Peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Couldn't send request to peer 0f84a67fab33464bb93a32fd5b934b8d. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:22.305 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.37.2:37155
00:28:22.307 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.308755  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/logs
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.2:37155
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.2
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33733
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.2
00:28:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:22.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:22.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:22.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:22.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.608961  2740 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:22.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.609421  2740 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:22.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.609867  2740 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.641904  2740 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.642709  2740 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.2
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.678045  2740 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.2:37155
00:28:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.2
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33733
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.2
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/logs
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:28:22.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:22.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.679715  2740 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:22.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:22.681356  2740 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.693724  2748 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:22.696717  2749 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:23.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.891793  2740 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.197s	user 0.454s	sys 0.739s
00:28:23.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.892151  2740 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.197s	user 0.454s	sys 0.739s
00:28:23.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.893664  2740 server_base.cc:1061] running on GCE node
00:28:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:23.895226  2757 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.896498  2740 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:23.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.898643  2740 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:23.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.902104  2740 hybrid_clock.cc:648] HybridClock initialized: now 1783470503902048 us; error 32 us; skew 500 ppm
00:28:23.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.903182  2740 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:23.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.911399  2740 webserver.cc:534] Webserver started at http://127.1.37.2:33733/ using document root <none> and password file <none>
00:28:23.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.912563  2740 fs_manager.cc:362] Metadata directory not provided
00:28:23.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.912825  2740 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:23.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.922750  2740 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.002s	sys 0.002s
00:28:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.927424  2762 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.928611  2740 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
00:28:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.928901  2740 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f84a67fab33464bb93a32fd5b934b8d"
00:28:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:53 on dist-test-slave-4l7f"
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:23.930778  2740 fs_report.cc:389] FS layout report
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/data
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:24.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.003835  2740 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.005323  2740 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:24.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.005726  2740 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:24.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.008291  2740 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:24.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.013717  2769 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:28:24.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.021802  2740 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:24.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.022032  2740 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
00:28:24.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.022329  2740 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:24.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.026862  2740 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:24.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.027045  2740 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
00:28:24.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.027503  2769 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Bootstrap starting.
00:28:24.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.083099  2769 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Log is configured to *not* fsync() on all Append() calls
00:28:24.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.149627  2769 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:24.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.150521  2769 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Bootstrap complete.
00:28:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.152156  2769 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Time spent bootstrapping tablet: real 0.125s	user 0.099s	sys 0.021s
00:28:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.170485  2769 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.172984  2769 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f84a67fab33464bb93a32fd5b934b8d, State: Initialized, Role: FOLLOWER
00:28:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.173969  2769 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:24.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.178292  2769 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Time spent starting tablet: real 0.026s	user 0.017s	sys 0.007s
00:28:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.187722  2740 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.2:37155
00:28:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.188094  2877 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.2:37155 every 8 connection(s)
00:28:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.190054  2740 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:28:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.190802  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 2740
00:28:24.192 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:24.213 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:24.214 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.37.3:36659
00:28:24.215 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:24.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.215555  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 2365
00:28:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.237776  2878 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:28:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.238276  2878 heartbeater.cc:461] Registering TS with master...
00:28:24.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.239605  2878 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:28:24.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.244019  1371 ts_manager.cc:194] Re-registered known tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:24.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.246593  2879 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:28:24.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.246773  1371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.2:47947
00:28:24.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.246932  2879 heartbeater.cc:461] Registering TS with master...
00:28:24.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.247745  2879 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:28:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.251293  1292 ts_manager.cc:194] Re-registered known tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:24.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.258482  2883 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:28:24.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.258781  2883 heartbeater.cc:461] Registering TS with master...
00:28:24.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.259392  2883 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:28:24.260 [DEBUG - kudu-nio-2] (Connection.java:239) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] handling channelInactive
00:28:24.261 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state READY due to: connection closed
00:28:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:24.262264  1216 ts_manager.cc:194] Re-registered known tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:24.263 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:24.264 [DEBUG - Time-limited test] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:24.265 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.265 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.265 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.286 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:24.287 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:24.287 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [1ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [22ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@873687341(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@369e6ae3 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77d6960c -> wakeup thread Time-limited test)) after lookup
00:28:24.288 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:24.289 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.289 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.289 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.307 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:24.307 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:24.307 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [1ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [22ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [23ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [25ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [25ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@873687341(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@369e6ae3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1892893549 after retry RPC@871816589), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77d6960c -> wakeup thread Time-limited test -> (continuation of Deferred@1892893549 after retry RPC@871816589))) after lookup
00:28:24.308 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:24.309 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.309 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.309 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.347 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:24.347 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:24.348 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [1ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [22ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [23ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [25ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [25ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [44ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [45ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [45ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [82ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@873687341(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@369e6ae3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1892893549 after retry RPC@871816589) -> (continuation of Deferred@1558648720 after retry RPC@1501078324), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77d6960c -> wakeup thread Time-limited test -> (continuation of Deferred@1892893549 after retry RPC@871816589) -> (continuation of Deferred@1558648720 after retry RPC@1501078324))) after lookup
00:28:24.348 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:24.349 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.349 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.349 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.387 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:24.387 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:24.388 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=124), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [1ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [22ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [23ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [25ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [25ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [44ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [45ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [45ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [82ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [84ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [85ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [85ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [119ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [123ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@873687341(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@369e6ae3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1892893549 after retry RPC@871816589) -> (continuation of Deferred@1558648720 after retry RPC@1501078324) -> (continuation of Deferred@1290927356 after retry RPC@1342998357), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77d6960c -> wakeup thread Time-limited test -> (continuation of Deferred@1892893549 after retry RPC@871816589) -> (continuation of Deferred@1558648720 after retry RPC@1501078324) -> (continuation of Deferred@1290927356 after retry RPC@1342998357))) after lookup
00:28:24.388 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:24.389 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.389 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.389 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.546 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:24.547 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:24.547 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=283), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [1ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [22ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [23ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [25ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [25ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [44ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [45ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [45ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [82ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [84ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [85ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [85ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [119ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [123ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [124ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [125ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [125ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [279ms] refreshing cache from master, [279ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [282ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@873687341(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@369e6ae3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1892893549 after retry RPC@871816589) -> (continuation of Deferred@1558648720 after retry RPC@1501078324) -> (continuation of Deferred@1290927356 after retry RPC@1342998357) -> (continuation of Deferred@1827142814 after retry RPC@1491462264), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77d6960c -> wakeup thread Time-limited test -> (continuation of Deferred@1892893549 after retry RPC@871816589) -> (continuation of Deferred@1558648720 after retry RPC@1501078324) -> (continuation of Deferred@1290927356 after retry RPC@1342998357) -> (continuation of Deferred@1827142814 after retry RPC@1491462264))) after lookup
00:28:24.548 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:24.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:24.549 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.232010  2890 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 6 FOLLOWER]: Starting pre-election (detected failure of leader d7f44bee9e9544549e3b9aebe7adbdfe)
00:28:25.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.232441  2890 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:25.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.234122  2890 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:25.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.239341  2581 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111)
00:28:25.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.245260  2832 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "0f84a67fab33464bb93a32fd5b934b8d" is_pre_election: true
00:28:25.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.245990  2832 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d0f7154f76e840bc92259656e7fd7d0b in term 5.
00:28:25.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.247185  2580 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d0f7154f76e840bc92259656e7fd7d0b; no voters: d7f44bee9e9544549e3b9aebe7adbdfe
00:28:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.247787  2890 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 6 FOLLOWER]: Leader pre-election won for term 7
00:28:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.248027  2890 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 6 FOLLOWER]: Starting leader election (detected failure of leader d7f44bee9e9544549e3b9aebe7adbdfe)
00:28:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.248247  2890 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 6 FOLLOWER]: Advancing to term 7
00:28:25.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.252031  2878 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:28:25.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.252863  2890 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.255668  2832 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "0f84a67fab33464bb93a32fd5b934b8d"
00:28:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.256191  2832 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 5 FOLLOWER]: Advancing to term 7
00:28:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.257493  2890 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 7 election: Requested vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.258613  2581 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111)
00:28:25.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.262770  2832 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d0f7154f76e840bc92259656e7fd7d0b in term 7.
00:28:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.263554  2580 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d0f7154f76e840bc92259656e7fd7d0b; no voters: d7f44bee9e9544549e3b9aebe7adbdfe
00:28:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.264245  2890 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 7 FOLLOWER]: Leader election won for term 7
00:28:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.265496  2890 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 7 LEADER]: Becoming Leader. State: Replica: d0f7154f76e840bc92259656e7fd7d0b, State: Running, Role: LEADER
00:28:25.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.266395  2890 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.273867  1371 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b reported cstate change: term changed from 6 to 7, leader changed from d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3) to d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1). New cstate: current_term: 7 leader_uuid: "d0f7154f76e840bc92259656e7fd7d0b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: HEALTHY } } }
00:28:25.407 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:25.407 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:25.408 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1144), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [1ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [22ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [23ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [25ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [25ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [39ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [42ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [44ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [45ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [45ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [79ms] refreshing cache from master, [79ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [82ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [84ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [85ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [85ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [119ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [123ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [124ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [125ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [125ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [279ms] refreshing cache from master, [279ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [282ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [283ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [285ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [285ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1139ms] refreshing cache from master, [1139ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [1143ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@873687341(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@369e6ae3 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1892893549 after retry RPC@871816589) -> (continuation of Deferred@1558648720 after retry RPC@1501078324) -> (continuation of Deferred@1290927356 after retry RPC@1342998357) -> (continuation of Deferred@1827142814 after retry RPC@1491462264) -> (continuation of Deferred@1057937633 after retry RPC@1796945106), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@77d6960c -> wakeup thread Time-limited test -> (continuation of Deferred@1892893549 after retry RPC@871816589) -> (continuation of Deferred@1558648720 after retry RPC@1501078324) -> (continuation of Deferred@1290927356 after retry RPC@1342998357) -> (continuation of Deferred@1827142814 after retry RPC@1491462264) -> (continuation of Deferred@1057937633 after retry RPC@1796945106))) after lookup
00:28:25.408 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:25.409 [DEBUG - kudu-nio-1] (Connection.java:757) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] Successfully connected to peer
00:28:25.430 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xff7c1dc6, L:/127.0.0.1:48772 - R:/127.1.37.1:36133] using TOKEN/null
00:28:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.439673  2832 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 7 FOLLOWER]: Refusing update from remote peer d0f7154f76e840bc92259656e7fd7d0b: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 26. (index mismatch)
00:28:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.441236  2895 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
00:28:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.441897  2581 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b -> Peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Couldn't send request to peer d7f44bee9e9544549e3b9aebe7adbdfe. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.479732  2911 mvcc.cc:204] Tried to move back new op lower bound from 7305095190266167296 to 7305095189578731520. Current Snapshot: MvccSnapshot[applied={T|T < 7305095190266167296}]
00:28:25.503 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.37.3:36659
00:28:25.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.504513  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:25.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/logs
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.3:36659
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.3
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44545
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.3
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:25.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.816957  2918 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:25.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.817417  2918 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.817881  2918 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.850306  2918 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:25.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.851190  2918 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.3
00:28:25.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.886510  2918 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.3:36659
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.3
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44545
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.3
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/logs
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:25.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:25.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:25.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:25.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:25.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:28:25.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:28:25.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:25.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.888209  2918 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:25.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:25.889860  2918 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.901443  2924 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:26.989872  2581 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111) [suppressed 5 similar messages]
00:28:27.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:25.902237  2925 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:27.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.261365  2927 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:27.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.263520  2926 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1356 milliseconds
00:28:27.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.264233  2918 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.362s	user 0.458s	sys 0.843s
00:28:27.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:27.264595  2918 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.363s	user 0.462s	sys 0.843s
00:28:27.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.264873  2918 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:27.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.266307  2918 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.268931  2918 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.270354  2918 hybrid_clock.cc:648] HybridClock initialized: now 1783470507270311 us; error 35 us; skew 500 ppm
00:28:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.271399  2918 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:27.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.279637  2918 webserver.cc:534] Webserver started at http://127.1.37.3:44545/ using document root <none> and password file <none>
00:28:27.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.280949  2918 fs_manager.cc:362] Metadata directory not provided
00:28:27.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.281239  2918 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.292425  2918 fs_manager.cc:714] Time spent opening directory manager: real 0.007s	user 0.001s	sys 0.005s
00:28:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.298229  2937 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:27.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.299654  2918 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
00:28:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.300060  2918 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:28:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:55 on dist-test-slave-4l7f"
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.302740  2918 fs_report.cc:389] FS layout report
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/data
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:27.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.414729  2918 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:27.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.416173  2918 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:27.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.416605  2918 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:27.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.419117  2918 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:27.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.424760  2946 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:28:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.433697  2918 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.433908  2918 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.011s	user 0.000s	sys 0.002s
00:28:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.434115  2918 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:27.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.438642  2918 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:27.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.438827  2918 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.000s	sys 0.003s
00:28:27.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.439250  2946 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Bootstrap starting.
00:28:27.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.500129  2946 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Log is configured to *not* fsync() on all Append() calls
00:28:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.595307  2946 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.596362  2946 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Bootstrap complete.
00:28:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.598011  2946 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Time spent bootstrapping tablet: real 0.159s	user 0.115s	sys 0.038s
00:28:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.602870  2918 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.3:36659
00:28:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.602972  3053 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.3:36659 every 8 connection(s)
00:28:27.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.606292  2918 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:28:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.616221  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 2918
00:28:27.617 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.620483  2946 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.623427  2946 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: d7f44bee9e9544549e3b9aebe7adbdfe, State: Initialized, Role: FOLLOWER
00:28:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.624542  2946 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:27.644 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:27.645 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.37.1:36133
00:28:27.646 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:27.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.646451  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 2559
00:28:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.662672  2946 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Time spent starting tablet: real 0.064s	user 0.036s	sys 0.027s
00:28:27.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.679401  3055 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:28:27.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.679919  3055 heartbeater.cc:461] Registering TS with master...
00:28:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.681186  3055 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:28:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.684376  3056 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:28:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.684748  3056 heartbeater.cc:461] Registering TS with master...
00:28:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.685631  3056 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:28:27.686 [DEBUG - kudu-nio-1] (Connection.java:239) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] handling channelInactive
00:28:27.686 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state READY due to: connection closed
00:28:27.688 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:27.689 [DEBUG - Time-limited test] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:27.691 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.691 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.691 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.689059  1217 ts_manager.cc:194] Re-registered known tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:28:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.693013  3054 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:28:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.693333  3054 heartbeater.cc:461] Registering TS with master...
00:28:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.694151  3054 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:28:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.694314  1292 ts_manager.cc:194] Re-registered known tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:28:27.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.697084  1371 ts_manager.cc:194] Re-registered known tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:28:27.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.700021  1371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.3:34365
00:28:27.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:27.703044  3054 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:28:27.709 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:27.709 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:27.710 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1507053704(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@6b5cd2c4 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@16c723b1 -> wakeup thread Time-limited test)) after lookup
00:28:27.710 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:27.711 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.711 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.712 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.726 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:27.726 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:27.727 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [22ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [23ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1507053704(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@6b5cd2c4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@16c723b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229))) after lookup
00:28:27.727 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:27.728 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.728 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.728 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.746 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:27.747 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:27.747 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=58), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [22ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [23ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [38ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [39ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [39ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [57ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1507053704(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@6b5cd2c4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229) -> (continuation of Deferred@1787624013 after retry RPC@952081945), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@16c723b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229) -> (continuation of Deferred@1787624013 after retry RPC@952081945))) after lookup
00:28:27.747 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:27.748 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.748 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.749 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.786 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:27.786 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:27.786 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=97), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [22ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [23ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [38ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [39ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [39ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [57ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [58ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [59ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [60ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [97ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1507053704(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@6b5cd2c4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229) -> (continuation of Deferred@1787624013 after retry RPC@952081945) -> (continuation of Deferred@2016481825 after retry RPC@1010435176), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@16c723b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229) -> (continuation of Deferred@1787624013 after retry RPC@952081945) -> (continuation of Deferred@2016481825 after retry RPC@1010435176))) after lookup
00:28:27.787 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:27.788 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.788 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:27.788 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:28.066 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:28.067 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:28.067 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=378), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [22ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [23ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [38ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [39ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [39ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [57ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [58ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [59ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [60ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [97ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [98ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [99ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [99ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [374ms] refreshing cache from master, [374ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [377ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1507053704(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@6b5cd2c4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229) -> (continuation of Deferred@1787624013 after retry RPC@952081945) -> (continuation of Deferred@2016481825 after retry RPC@1010435176) -> (continuation of Deferred@982714968 after retry RPC@1449545766), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@16c723b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229) -> (continuation of Deferred@1787624013 after retry RPC@952081945) -> (continuation of Deferred@2016481825 after retry RPC@1010435176) -> (continuation of Deferred@982714968 after retry RPC@1449545766))) after lookup
00:28:28.068 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:28.069 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:28.069 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:28.069 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:28.186 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:28.187 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:28.187 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=498), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [22ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [23ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [38ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [39ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [39ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [57ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [58ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [59ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [60ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [97ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [98ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [99ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [99ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [374ms] refreshing cache from master, [374ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [377ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [378ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [380ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [380ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [494ms] refreshing cache from master, [494ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [497ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1507053704(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@6b5cd2c4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229) -> (continuation of Deferred@1787624013 after retry RPC@952081945) -> (continuation of Deferred@2016481825 after retry RPC@1010435176) -> (continuation of Deferred@982714968 after retry RPC@1449545766) -> (continuation of Deferred@173085900 after retry RPC@1850905280), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@16c723b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229) -> (continuation of Deferred@1787624013 after retry RPC@952081945) -> (continuation of Deferred@2016481825 after retry RPC@1010435176) -> (continuation of Deferred@982714968 after retry RPC@1449545766) -> (continuation of Deferred@173085900 after retry RPC@1850905280))) after lookup
00:28:28.188 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:28.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:28.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:28.189 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.927717  3067 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 7 FOLLOWER]: Starting pre-election (detected failure of leader d0f7154f76e840bc92259656e7fd7d0b)
00:28:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.928123  3067 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:28.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.930701  3067 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:28.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:28.940814  2763 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:28:28.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:28.944092  2763 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:28:28.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.948339  3009 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "0f84a67fab33464bb93a32fd5b934b8d" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" is_pre_election: true
00:28:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.949028  3009 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f84a67fab33464bb93a32fd5b934b8d in term 6.
00:28:28.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.950250  2765 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: d0f7154f76e840bc92259656e7fd7d0b
00:28:28.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.950901  3067 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 7 FOLLOWER]: Leader pre-election won for term 8
00:28:28.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.951159  3067 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 7 FOLLOWER]: Starting leader election (detected failure of leader d0f7154f76e840bc92259656e7fd7d0b)
00:28:28.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.951377  3067 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 7 FOLLOWER]: Advancing to term 8
00:28:28.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.955616  3067 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.957059  3067 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 8 election: Requested vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:28.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.957729  3009 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "0f84a67fab33464bb93a32fd5b934b8d" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:28:28.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.958241  3009 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 6 FOLLOWER]: Advancing to term 8
00:28:28.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:28.960726  2763 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:28:28.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.965114  3009 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f84a67fab33464bb93a32fd5b934b8d in term 8.
00:28:28.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.965854  2765 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: d0f7154f76e840bc92259656e7fd7d0b
00:28:28.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.966446  3067 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 8 FOLLOWER]: Leader election won for term 8
00:28:28.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.967873  3067 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 8 LEADER]: Becoming Leader. State: Replica: 0f84a67fab33464bb93a32fd5b934b8d, State: Running, Role: LEADER
00:28:28.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.968570  3067 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:28.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:28.976738  1372 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d reported cstate change: term changed from 7 to 8, leader changed from d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1) to 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2). New cstate: current_term: 8 leader_uuid: "0f84a67fab33464bb93a32fd5b934b8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: UNKNOWN } } }
00:28:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:29.397862  2763 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d -> Peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Couldn't send request to peer d0f7154f76e840bc92259656e7fd7d0b. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:29.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.563200  3009 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 8 FOLLOWER]: Refusing update from remote peer 0f84a67fab33464bb93a32fd5b934b8d: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
00:28:29.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.564630  3067 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
00:28:29.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:29.577952  3085 mvcc.cc:204] Tried to move back new op lower bound from 7305095190266167296 to 7305095189578731520. Current Snapshot: MvccSnapshot[applied={T|T < 7305095190266167296}]
00:28:30.307 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:30.307 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:30.308 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=14, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=2618), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [22ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [23ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [37ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [38ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [39ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [39ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [54ms] refreshing cache from master, [54ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [57ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [58ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [59ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [60ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [94ms] refreshing cache from master, [94ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [97ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [98ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [99ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [99ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [374ms] refreshing cache from master, [374ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [377ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [378ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [380ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [380ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [494ms] refreshing cache from master, [494ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [497ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [499ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [500ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [500ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2614ms] refreshing cache from master, [2614ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [2617ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1507053704(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@6b5cd2c4 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229) -> (continuation of Deferred@1787624013 after retry RPC@952081945) -> (continuation of Deferred@2016481825 after retry RPC@1010435176) -> (continuation of Deferred@982714968 after retry RPC@1449545766) -> (continuation of Deferred@173085900 after retry RPC@1850905280) -> (continuation of Deferred@279095981 after retry RPC@1397127527), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@16c723b1 -> wakeup thread Time-limited test -> (continuation of Deferred@1376816085 after retry RPC@2112217229) -> (continuation of Deferred@1787624013 after retry RPC@952081945) -> (continuation of Deferred@2016481825 after retry RPC@1010435176) -> (continuation of Deferred@982714968 after retry RPC@1449545766) -> (continuation of Deferred@173085900 after retry RPC@1850905280) -> (continuation of Deferred@279095981 after retry RPC@1397127527))) after lookup
00:28:30.308 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:30.309 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] Successfully connected to peer
00:28:30.326 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x51feb99c, L:/127.0.0.1:34446 - R:/127.1.37.2:37155] using TOKEN/null
00:28:30.406 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.37.1:36133
00:28:30.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:30.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.408293  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:30.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:30.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:30.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data
00:28:30.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:30.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:30.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/logs
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.1:36133
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.1
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44645
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.1
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:30.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:30.738842  3099 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:30.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:30.739295  3099 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:30.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:30.739729  3099 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:30.770948  3099 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:30.771756  3099 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.1
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.806139  3099 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.1:36133
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.1
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44645
00:28:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.1
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/logs
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.807411  3099 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:30.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:30.809266  3099 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:30.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:30.821484  3107 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:31.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:31.765115  2763 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d -> Peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Couldn't send request to peer d0f7154f76e840bc92259656e7fd7d0b. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:28:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:31.986140  1248 debug-util.cc:398] Leaking SignalData structure 0x7b08000b12e0 after lost signal to thread 1185
00:28:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:31.986836  1248 debug-util.cc:398] Leaking SignalData structure 0x7b0800050480 after lost signal to thread 1251
00:28:32.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:32.217557  3110 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:32.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:32.219224  3109 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1393 milliseconds
00:28:32.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:30.823107  3108 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:32.222613  3099 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.399s	user 0.482s	sys 0.853s
00:28:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:32.222939  3099 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.400s	user 0.482s	sys 0.853s
00:28:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.223187  3099 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.225797  3099 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:32.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.234860  3099 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.236331  3099 hybrid_clock.cc:648] HybridClock initialized: now 1783470512236187 us; error 143 us; skew 500 ppm
00:28:32.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.237437  3099 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.245829  3099 webserver.cc:534] Webserver started at http://127.1.37.1:44645/ using document root <none> and password file <none>
00:28:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.246974  3099 fs_manager.cc:362] Metadata directory not provided
00:28:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.247227  3099 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:32.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.257998  3099 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.007s	sys 0.001s
00:28:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.263870  3119 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:32.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.265193  3099 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
00:28:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.265595  3099 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0f7154f76e840bc92259656e7fd7d0b"
00:28:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:51 on dist-test-slave-4l7f"
00:28:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.268247  3099 fs_report.cc:389] FS layout report
00:28:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/data
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:32.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.369395  3099 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.370939  3099 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.371418  3099 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:32.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.374498  3099 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.381004  3126 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:28:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.388954  3099 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.389176  3099 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.000s	sys 0.002s
00:28:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.389505  3099 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.394388  3099 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.394634  3099 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.001s
00:28:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.395002  3126 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap starting.
00:28:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.465946  3126 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Log is configured to *not* fsync() on all Append() calls
00:28:32.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.566001  3126 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:32.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.566943  3126 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap complete.
00:28:32.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.568580  3126 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Time spent bootstrapping tablet: real 0.174s	user 0.130s	sys 0.039s
00:28:32.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.589319  3126 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:32.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.592790  3126 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: d0f7154f76e840bc92259656e7fd7d0b, State: Initialized, Role: FOLLOWER
00:28:32.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.594053  3126 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.617951  3126 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Time spent starting tablet: real 0.049s	user 0.025s	sys 0.004s
00:28:32.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.659482  3099 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.1:36133
00:28:32.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.659694  3236 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.1:36133 every 8 connection(s)
00:28:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.662887  3099 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:32.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.673275  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 3099
00:28:32.674 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.712086  3237 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:28:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.712540  3237 heartbeater.cc:461] Registering TS with master...
00:28:32.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.714125  3237 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:28:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.719396  1372 ts_manager.cc:194] Re-registered known tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:32.721 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:32.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.722090  1372 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.1:60601
00:28:32.723 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.37.2:37155
00:28:32.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:32.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.723902  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 2740
00:28:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.733779  3238 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:28:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.734143  3238 heartbeater.cc:461] Registering TS with master...
00:28:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.734737  3238 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:28:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.735971  3239 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:28:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.736307  3239 heartbeater.cc:461] Registering TS with master...
00:28:32.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.736918  3239 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:28:32.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.739107  1292 ts_manager.cc:194] Re-registered known tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:32.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:32.740808  1216 ts_manager.cc:194] Re-registered known tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:32.765 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] handling channelInactive
00:28:32.765 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state READY due to: connection closed
00:28:32.766 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:32.766 [DEBUG - Time-limited test] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:32.768 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.768 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.768 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.786 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:32.786 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:32.786 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=20), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1592016438(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@3e89e9cc -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1638322a -> wakeup thread Time-limited test)) after lookup
00:28:32.787 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:32.788 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.788 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.788 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.806 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:32.806 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:32.807 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=40), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [22ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1592016438(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@3e89e9cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1638322a -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069))) after lookup
00:28:32.807 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:32.810 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.810 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.810 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.826 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:32.826 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:32.827 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=60), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [22ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [44ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [44ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [60ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1592016438(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@3e89e9cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069) -> (continuation of Deferred@1023497558 after retry RPC@779221275), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1638322a -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069) -> (continuation of Deferred@1023497558 after retry RPC@779221275))) after lookup
00:28:32.827 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:32.828 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.828 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.828 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.966 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:32.967 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:32.967 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=201), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [22ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [44ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [44ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [60ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [61ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [62ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [62ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [197ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [200ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1592016438(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@3e89e9cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069) -> (continuation of Deferred@1023497558 after retry RPC@779221275) -> (continuation of Deferred@1941330878 after retry RPC@1387191794), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1638322a -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069) -> (continuation of Deferred@1023497558 after retry RPC@779221275) -> (continuation of Deferred@1941330878 after retry RPC@1387191794))) after lookup
00:28:32.968 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:32.969 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.969 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:32.969 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:33.167 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:33.167 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:33.168 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=401), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [22ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [44ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [44ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [60ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [61ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [62ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [62ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [197ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [200ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [201ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [203ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [203ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [397ms] refreshing cache from master, [398ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [401ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1592016438(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@3e89e9cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069) -> (continuation of Deferred@1023497558 after retry RPC@779221275) -> (continuation of Deferred@1941330878 after retry RPC@1387191794) -> (continuation of Deferred@1442877205 after retry RPC@1919261034), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1638322a -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069) -> (continuation of Deferred@1023497558 after retry RPC@779221275) -> (continuation of Deferred@1941330878 after retry RPC@1387191794) -> (continuation of Deferred@1442877205 after retry RPC@1919261034))) after lookup
00:28:33.168 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:33.169 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:33.169 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:33.169 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:33.206 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:33.207 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:33.207 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=440), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [22ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [44ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [44ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [60ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [61ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [62ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [62ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [197ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [200ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [201ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [203ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [203ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [397ms] refreshing cache from master, [398ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [401ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [402ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [403ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [403ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [437ms] refreshing cache from master, [437ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [440ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1592016438(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@3e89e9cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069) -> (continuation of Deferred@1023497558 after retry RPC@779221275) -> (continuation of Deferred@1941330878 after retry RPC@1387191794) -> (continuation of Deferred@1442877205 after retry RPC@1919261034) -> (continuation of Deferred@541331525 after retry RPC@2117362670), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1638322a -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069) -> (continuation of Deferred@1023497558 after retry RPC@779221275) -> (continuation of Deferred@1941330878 after retry RPC@1387191794) -> (continuation of Deferred@1442877205 after retry RPC@1919261034) -> (continuation of Deferred@541331525 after retry RPC@2117362670))) after lookup
00:28:33.207 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:33.208 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:33.208 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state CONNECTING due to: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:33.209 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155
00:28:33.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.725545  3237 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:28:33.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.881398  3249 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 7 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:33.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.881832  3249 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:33.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.884023  3249 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:33.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.887915  3121 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:33.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.891656  3121 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:33.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.894639  3008 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" is_pre_election: true
00:28:33.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.895975  3122 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d0f7154f76e840bc92259656e7fd7d0b; no voters: 0f84a67fab33464bb93a32fd5b934b8d, d7f44bee9e9544549e3b9aebe7adbdfe
00:28:33.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.896590  3249 raft_consensus.cc:2749] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 7 FOLLOWER]: Leader pre-election lost for term 8. Reason: could not achieve majority
00:28:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.916577  3254 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 0f84a67fab33464bb93a32fd5b934b8d)
00:28:33.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.916922  3254 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:33.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.919096  3254 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.923019  2941 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:33.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.931202  2941 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:33.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.935606  3189 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "d0f7154f76e840bc92259656e7fd7d0b" is_pre_election: true
00:28:33.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.936435  3189 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d7f44bee9e9544549e3b9aebe7adbdfe in term 7.
00:28:33.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.937872  2940 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d0f7154f76e840bc92259656e7fd7d0b, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: 0f84a67fab33464bb93a32fd5b934b8d
00:28:33.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.938616  3254 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 8 FOLLOWER]: Leader pre-election won for term 9
00:28:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.938925  3254 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 8 FOLLOWER]: Starting leader election (detected failure of leader 0f84a67fab33464bb93a32fd5b934b8d)
00:28:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.939230  3254 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 8 FOLLOWER]: Advancing to term 9
00:28:33.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.945451  3254 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:33.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.947199  3254 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 9 election: Requested vote from peers 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:33.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.948364  3189 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "d0f7154f76e840bc92259656e7fd7d0b"
00:28:33.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.948900  3189 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 7 FOLLOWER]: Advancing to term 9
00:28:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:33.950482  2941 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111)
00:28:33.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.955474  3189 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d7f44bee9e9544549e3b9aebe7adbdfe in term 9.
00:28:33.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.956269  2940 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d0f7154f76e840bc92259656e7fd7d0b, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: 0f84a67fab33464bb93a32fd5b934b8d
00:28:33.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.956833  3254 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 9 FOLLOWER]: Leader election won for term 9
00:28:33.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.958274  3254 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 9 LEADER]: Becoming Leader. State: Replica: d7f44bee9e9544549e3b9aebe7adbdfe, State: Running, Role: LEADER
00:28:33.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.959467  3254 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:33.966972  1372 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe reported cstate change: term changed from 8 to 9, leader changed from 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2) to d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3). New cstate: current_term: 9 leader_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: UNKNOWN } } }
00:28:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:34.583045  3189 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 9 FOLLOWER]: Refusing update from remote peer d7f44bee9e9544549e3b9aebe7adbdfe: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
00:28:34.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:34.584697  3272 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [LEADER]: Connected to new peer: Peer: permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
00:28:34.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:34.586597  2941 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe -> Peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Couldn't send request to peer 0f84a67fab33464bb93a32fd5b934b8d. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:36.769 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:36.769 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:36.769 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=14, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=4003), Traces: [0ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [2ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [2ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [17ms] refreshing cache from master, [17ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [20ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [21ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [22ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [22ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [37ms] refreshing cache from master, [37ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [40ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [41ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [44ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [44ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [57ms] refreshing cache from master, [57ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [60ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [61ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [62ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [62ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [197ms] refreshing cache from master, [197ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [200ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [201ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [203ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [203ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [397ms] refreshing cache from master, [398ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [401ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [402ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [403ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [403ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [437ms] refreshing cache from master, [437ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [440ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [441ms] sending RPC to server 0f84a67fab33464bb93a32fd5b934b8d, [442ms] received response from server 0f84a67fab33464bb93a32fd5b934b8d: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [443ms] delaying RPC due to: Network error: Failed to connect to peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155): Connection refused: /127.1.37.2:37155, [3997ms] refreshing cache from master, [3997ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [4002ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1592016438(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@3e89e9cc -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069) -> (continuation of Deferred@1023497558 after retry RPC@779221275) -> (continuation of Deferred@1941330878 after retry RPC@1387191794) -> (continuation of Deferred@1442877205 after retry RPC@1919261034) -> (continuation of Deferred@541331525 after retry RPC@2117362670) -> (continuation of Deferred@1591195390 after retry RPC@1554036373), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1638322a -> wakeup thread Time-limited test -> (continuation of Deferred@1930966430 after retry RPC@610726069) -> (continuation of Deferred@1023497558 after retry RPC@779221275) -> (continuation of Deferred@1941330878 after retry RPC@1387191794) -> (continuation of Deferred@1442877205 after retry RPC@1919261034) -> (continuation of Deferred@541331525 after retry RPC@2117362670) -> (continuation of Deferred@1591195390 after retry RPC@1554036373))) after lookup
00:28:36.770 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:36.771 [DEBUG - kudu-nio-1] (Connection.java:757) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] Successfully connected to peer
00:28:36.792 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xb1c2445b, L:/127.0.0.1:47130 - R:/127.1.37.3:36659] using TOKEN/null
00:28:36.836 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.37.2:37155
00:28:36.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:36.838080  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/logs
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.2:37155
00:28:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.2
00:28:36.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:36.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33733
00:28:36.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.2
00:28:36.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:36.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:36.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:36.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.099805  2941 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe -> Peer 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155): Couldn't send request to peer 0f84a67fab33464bb93a32fd5b934b8d. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.2:37155: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:28:37.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.133661  3300 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:37.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.134120  3300 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:37.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.134581  3300 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:37.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.166378  3300 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.167187  3300 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.2
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.202113  3300 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.2:37155
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.2
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33733
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.2
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/logs
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.203449  3300 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:37.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.205147  3300 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.217648  3308 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:37.218233  3309 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:37.221902  3300 server_base.cc:1061] running on GCE node
00:28:38.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:38.319056  3311 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.321437  3300 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.324040  3300 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.325439  3300 hybrid_clock.cc:648] HybridClock initialized: now 1783470518325392 us; error 45 us; skew 500 ppm
00:28:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.326203  3300 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.332103  3300 webserver.cc:534] Webserver started at http://127.1.37.2:33733/ using document root <none> and password file <none>
00:28:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.333000  3300 fs_manager.cc:362] Metadata directory not provided
00:28:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.333184  3300 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.340921  3300 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.000s
00:28:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.345772  3320 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.346874  3300 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.003s
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.347172  3300 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f84a67fab33464bb93a32fd5b934b8d"
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:53 on dist-test-slave-4l7f"
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.349097  3300 fs_report.cc:389] FS layout report
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/wal
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/data
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:38.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.398311  3300 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:38.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.399755  3300 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:38.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.400158  3300 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.402683  3300 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.408219  3327 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:28:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.416318  3300 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.416567  3300 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.002s	sys 0.000s
00:28:38.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.416850  3300 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:38.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.423712  3300 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:38.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.423961  3300 ts_tablet_manager.cc:595] Time spent register tablets: real 0.007s	user 0.004s	sys 0.000s
00:28:38.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.424311  3327 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Bootstrap starting.
00:28:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.486730  3327 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Log is configured to *not* fsync() on all Append() calls
00:28:38.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.578742  3327 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:38.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.579720  3327 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Bootstrap complete.
00:28:38.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.581336  3327 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Time spent bootstrapping tablet: real 0.157s	user 0.104s	sys 0.049s
00:28:38.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.592216  3300 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.2:37155
00:28:38.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.592415  3436 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.2:37155 every 8 connection(s)
00:28:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.594702  3300 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-1/data/info.pb
00:28:38.601 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.600258  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 3300
00:28:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.597671  3327 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.600760  3327 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f84a67fab33464bb93a32fd5b934b8d, State: Initialized, Role: FOLLOWER
00:28:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.601887  3327 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:38.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.619984  3327 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d: Time spent starting tablet: real 0.038s	user 0.023s	sys 0.005s
00:28:38.633 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:38.634 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.37.3:36659
00:28:38.634 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:38.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.634668  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 2918
00:28:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.656867  3439 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:28:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.657272  3437 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:28:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.657370  3439 heartbeater.cc:461] Registering TS with master...
00:28:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.657593  3437 heartbeater.cc:461] Registering TS with master...
00:28:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.658691  3439 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:28:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.658771  3437 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:28:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.659948  3438 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:28:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.660254  3438 heartbeater.cc:461] Registering TS with master...
00:28:38.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.661136  3438 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:28:38.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.663307  1217 ts_manager.cc:194] Re-registered known tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:38.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.665385  1292 ts_manager.cc:194] Re-registered known tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:38.670 [DEBUG - kudu-nio-1] (Connection.java:239) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] handling channelInactive
00:28:38.670 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state READY due to: connection closed
00:28:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.670689  1371 ts_manager.cc:194] Re-registered known tserver with Master: 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:38.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:38.673 [DEBUG - Time-limited test] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:38.675 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.675 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.675 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.677793  1371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.2:34481
00:28:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:38.680807  3437 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:28:38.687 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:38.687 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:38.687 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=14), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [14ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1396758774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@a8dd815 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3832085c -> wakeup thread Time-limited test)) after lookup
00:28:38.688 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:38.689 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.689 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.689 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.706 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:38.706 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:38.706 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [14ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [15ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [16ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [16ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1396758774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@a8dd815 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3832085c -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885))) after lookup
00:28:38.707 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:38.708 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.708 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.708 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.726 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:38.726 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:38.726 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=53), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [14ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [15ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [16ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [16ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [33ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [35ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [35ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [53ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1396758774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@a8dd815 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885) -> (continuation of Deferred@1380780037 after retry RPC@538170587), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3832085c -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885) -> (continuation of Deferred@1380780037 after retry RPC@538170587))) after lookup
00:28:38.727 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:38.728 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.728 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.728 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.766 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:38.766 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:38.767 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=93), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [14ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [15ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [16ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [16ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [33ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [35ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [35ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [53ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [54ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [55ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [55ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [89ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [93ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1396758774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@a8dd815 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885) -> (continuation of Deferred@1380780037 after retry RPC@538170587) -> (continuation of Deferred@248074932 after retry RPC@818084538), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3832085c -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885) -> (continuation of Deferred@1380780037 after retry RPC@538170587) -> (continuation of Deferred@248074932 after retry RPC@818084538))) after lookup
00:28:38.767 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:38.768 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.768 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.768 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.806 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:38.806 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:38.807 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=133), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [14ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [15ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [16ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [16ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [33ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [35ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [35ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [53ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [54ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [55ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [55ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [89ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [93ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [94ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [95ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [95ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [133ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1396758774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@a8dd815 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885) -> (continuation of Deferred@1380780037 after retry RPC@538170587) -> (continuation of Deferred@248074932 after retry RPC@818084538) -> (continuation of Deferred@2030911216 after retry RPC@1581063233), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3832085c -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885) -> (continuation of Deferred@1380780037 after retry RPC@538170587) -> (continuation of Deferred@248074932 after retry RPC@818084538) -> (continuation of Deferred@2030911216 after retry RPC@1581063233))) after lookup
00:28:38.807 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:38.808 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.808 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:38.808 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:39.686 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:39.687 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)[L]]}]
00:28:39.687 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1014), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [14ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [15ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [16ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [16ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [33ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [35ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [35ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [53ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [54ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [55ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [55ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [89ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [93ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [94ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [95ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [95ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [133ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [134ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [135ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [135ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1010ms] refreshing cache from master, [1010ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [1013ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1396758774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@a8dd815 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885) -> (continuation of Deferred@1380780037 after retry RPC@538170587) -> (continuation of Deferred@248074932 after retry RPC@818084538) -> (continuation of Deferred@2030911216 after retry RPC@1581063233) -> (continuation of Deferred@74373852 after retry RPC@847708420), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3832085c -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885) -> (continuation of Deferred@1380780037 after retry RPC@538170587) -> (continuation of Deferred@248074932 after retry RPC@818084538) -> (continuation of Deferred@2030911216 after retry RPC@1581063233) -> (continuation of Deferred@74373852 after retry RPC@847708420))) after lookup
00:28:39.688 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] connecting to peer
00:28:39.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:39.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] cleaning up while in state CONNECTING due to: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:39.690 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659
00:28:39.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.817829  3449 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 9 FOLLOWER]: Starting pre-election (detected failure of leader d7f44bee9e9544549e3b9aebe7adbdfe)
00:28:39.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.818223  3449 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.819887  3449 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:39.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:39.822415  3122 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111)
00:28:39.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:39.826577  3122 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111)
00:28:39.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.832794  3391 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "0f84a67fab33464bb93a32fd5b934b8d" is_pre_election: true
00:28:39.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.833513  3391 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d0f7154f76e840bc92259656e7fd7d0b in term 8.
00:28:39.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.834728  3121 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d0f7154f76e840bc92259656e7fd7d0b; no voters: d7f44bee9e9544549e3b9aebe7adbdfe
00:28:39.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.835399  3449 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 9 FOLLOWER]: Leader pre-election won for term 10
00:28:39.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.835742  3449 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 9 FOLLOWER]: Starting leader election (detected failure of leader d7f44bee9e9544549e3b9aebe7adbdfe)
00:28:39.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.836088  3449 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 9 FOLLOWER]: Advancing to term 10
00:28:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.841169  3449 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:39.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.842630  3449 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 10 election: Requested vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2:37155)
00:28:39.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.843766  3391 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "d0f7154f76e840bc92259656e7fd7d0b" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "0f84a67fab33464bb93a32fd5b934b8d"
00:28:39.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.844321  3391 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 8 FOLLOWER]: Advancing to term 10
00:28:39.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:39.846378  3122 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111)
00:28:39.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.850970  3391 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d0f7154f76e840bc92259656e7fd7d0b in term 10.
00:28:39.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.851783  3121 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d0f7154f76e840bc92259656e7fd7d0b; no voters: d7f44bee9e9544549e3b9aebe7adbdfe
00:28:39.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.852358  3449 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 10 FOLLOWER]: Leader election won for term 10
00:28:39.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.853417  3449 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 10 LEADER]: Becoming Leader. State: Replica: d0f7154f76e840bc92259656e7fd7d0b, State: Running, Role: LEADER
00:28:39.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.854285  3449 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:39.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:39.861397  1371 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b reported cstate change: term changed from 9 to 10, leader changed from d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3) to d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1). New cstate: current_term: 10 leader_uuid: "d0f7154f76e840bc92259656e7fd7d0b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: HEALTHY } } }
00:28:40.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:40.315848  3122 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b -> Peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Couldn't send request to peer d7f44bee9e9544549e3b9aebe7adbdfe. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:40.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.463274  3391 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 10 FOLLOWER]: Refusing update from remote peer d0f7154f76e840bc92259656e7fd7d0b: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
00:28:40.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:40.464607  3449 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
00:28:41.866 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:41.867 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:41.867 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=14, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=3194), Traces: [0ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [2ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [2ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [14ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [15ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [16ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [16ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [29ms] refreshing cache from master, [30ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [32ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [33ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [35ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [35ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [53ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [54ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [55ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [55ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [89ms] refreshing cache from master, [90ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [93ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [94ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [95ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [95ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [130ms] refreshing cache from master, [130ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [133ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [134ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [135ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [135ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1010ms] refreshing cache from master, [1010ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [1013ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [1015ms] sending RPC to server d7f44bee9e9544549e3b9aebe7adbdfe, [1016ms] received response from server d7f44bee9e9544549e3b9aebe7adbdfe: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [1017ms] delaying RPC due to: Network error: Failed to connect to peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659): Connection refused: /127.1.37.3:36659, [3190ms] refreshing cache from master, [3190ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [3193ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@1396758774(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@a8dd815 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885) -> (continuation of Deferred@1380780037 after retry RPC@538170587) -> (continuation of Deferred@248074932 after retry RPC@818084538) -> (continuation of Deferred@2030911216 after retry RPC@1581063233) -> (continuation of Deferred@74373852 after retry RPC@847708420) -> (continuation of Deferred@502724126 after retry RPC@681952500), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3832085c -> wakeup thread Time-limited test -> (continuation of Deferred@644288021 after retry RPC@247435885) -> (continuation of Deferred@1380780037 after retry RPC@538170587) -> (continuation of Deferred@248074932 after retry RPC@818084538) -> (continuation of Deferred@2030911216 after retry RPC@1581063233) -> (continuation of Deferred@74373852 after retry RPC@847708420) -> (continuation of Deferred@502724126 after retry RPC@681952500))) after lookup
00:28:41.868 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:41.869 [DEBUG - kudu-nio-1] (Connection.java:757) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] Successfully connected to peer
00:28:41.885 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x332981d8, L:/127.0.0.1:50908 - R:/127.1.37.1:36133] using TOKEN/null
00:28:41.928 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.37.3:36659
00:28:41.928 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:41.929697  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/logs
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.3:36659
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.3
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44545
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.3
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:41.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:42.220337  3486 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:42.220818  3486 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:42.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:42.221261  3486 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:42.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:42.252758  3486 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:42.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:42.253559  3486 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.3
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:42.289613  3486 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.3:36659
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.3
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44545
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.3
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/logs
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:28:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:42.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:42.290838  3486 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:42.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:42.292510  3486 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:42.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:42.305119  3492 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:42.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:42.994197  3122 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b -> Peer d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659): Couldn't send request to peer d7f44bee9e9544549e3b9aebe7adbdfe. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.3:36659: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
00:28:43.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:43.706849  3491 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 3486
00:28:43.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:43.851871  3491 kernel_stack_watchdog.cc:198] Thread 3486 stuck at /home/jenkins-slave/workspace/build_and_test@2/src/kudu/util/thread.cc:642 for 399ms:
00:28:43.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
00:28:43.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
00:28:43.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
00:28:43.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
00:28:43.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:43.852717  3486 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.548s	user 0.613s	sys 0.896s
00:28:43.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:43.853121  3486 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.548s	user 0.613s	sys 0.896s
00:28:43.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:42.305757  3493 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:43.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:43.853618  3494 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1546 milliseconds
00:28:43.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:43.854074  3495 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:43.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.854058  3486 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.858017  3486 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.859964  3486 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.861294  3486 hybrid_clock.cc:648] HybridClock initialized: now 1783470523861257 us; error 40 us; skew 500 ppm
00:28:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.862071  3486 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:43.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.867816  3486 webserver.cc:534] Webserver started at http://127.1.37.3:44545/ using document root <none> and password file <none>
00:28:43.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.868676  3486 fs_manager.cc:362] Metadata directory not provided
00:28:43.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.868857  3486 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:43.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.876022  3486 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.001s
00:28:43.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.880236  3508 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:43.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.881299  3486 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.000s	sys 0.003s
00:28:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.881656  3486 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:28:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:55 on dist-test-slave-4l7f"
00:28:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.883525  3486 fs_report.cc:389] FS layout report
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/wal
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/data
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:43.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.959731  3486 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.961217  3486 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.961649  3486 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:43.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.964109  3486 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.969545  3515 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:28:43.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.977393  3486 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:43.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.977626  3486 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.000s	sys 0.002s
00:28:43.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.977833  3486 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.982326  3486 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.982507  3486 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.003s	sys 0.000s
00:28:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:43.982935  3515 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Bootstrap starting.
00:28:44.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.047039  3515 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Log is configured to *not* fsync() on all Append() calls
00:28:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.137807  3515 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:44.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.138787  3515 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Bootstrap complete.
00:28:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.140005  3515 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Time spent bootstrapping tablet: real 0.157s	user 0.117s	sys 0.037s
00:28:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.151197  3486 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.3:36659
00:28:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.151358  3622 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.3:36659 every 8 connection(s)
00:28:44.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.153896  3486 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-2/data/info.pb
00:28:44.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.156772  3515 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.160089  3515 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: d7f44bee9e9544549e3b9aebe7adbdfe, State: Initialized, Role: FOLLOWER
00:28:44.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.161448  3515 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.164065  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 3486
00:28:44.165 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:44.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.194659  3515 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe: Time spent starting tablet: real 0.054s	user 0.023s	sys 0.017s
00:28:44.205 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:44.206 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.1.37.1:36133
00:28:44.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:44.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.206749  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 3099
00:28:44.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.213675  3623 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:28:44.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.214180  3623 heartbeater.cc:461] Registering TS with master...
00:28:44.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.214816  3624 heartbeater.cc:344] Connected to a master server at 127.1.37.61:35087
00:28:44.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.215175  3624 heartbeater.cc:461] Registering TS with master...
00:28:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.215483  3623 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:28:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.216548  3624 heartbeater.cc:507] Master 127.1.37.61:35087 requested a full tablet report, sending...
00:28:44.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.216955  3625 heartbeater.cc:344] Connected to a master server at 127.1.37.62:40671
00:28:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.217270  3625 heartbeater.cc:461] Registering TS with master...
00:28:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.218098  3625 heartbeater.cc:507] Master 127.1.37.62:40671 requested a full tablet report, sending...
00:28:44.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.220839  1371 ts_manager.cc:194] Re-registered known tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:28:44.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.221215  1292 ts_manager.cc:194] Re-registered known tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:28:44.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.222189  1217 ts_manager.cc:194] Re-registered known tserver with Master: d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659)
00:28:44.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.223366  1371 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.3:44437
00:28:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:44.226745  3623 heartbeater.cc:499] Master 127.1.37.60:42571 was elected leader, sending a full tablet report...
00:28:44.237 [DEBUG - kudu-nio-1] (Connection.java:239) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] handling channelInactive
00:28:44.237 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:44.237 [DEBUG - kudu-nio-1] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state READY due to: connection closed
00:28:44.238 [DEBUG - Time-limited test] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:44.239 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.239 [DEBUG - kudu-nio-2] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.239 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.246 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:44.247 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:44.247 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=9), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@632231478(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@232bf413 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ecc3ab5 -> wakeup thread Time-limited test)) after lookup
00:28:44.247 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:44.248 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.248 [DEBUG - kudu-nio-3] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.248 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.266 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:44.267 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:44.267 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [10ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@632231478(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@232bf413 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ecc3ab5 -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711))) after lookup
00:28:44.268 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:44.269 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.269 [DEBUG - kudu-nio-4] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.269 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.286 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:44.286 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:44.287 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [10ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [30ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [32ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [32ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [49ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@632231478(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@232bf413 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711) -> (continuation of Deferred@1816132298 after retry RPC@238329635), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ecc3ab5 -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711) -> (continuation of Deferred@1816132298 after retry RPC@238329635))) after lookup
00:28:44.287 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:44.288 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.288 [DEBUG - kudu-nio-5] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.288 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.406 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:44.406 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:44.407 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=169), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [10ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [30ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [32ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [32ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [49ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [50ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [51ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [51ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [166ms] refreshing cache from master, [166ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [169ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@632231478(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@232bf413 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711) -> (continuation of Deferred@1816132298 after retry RPC@238329635) -> (continuation of Deferred@1142118535 after retry RPC@1861491361), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ecc3ab5 -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711) -> (continuation of Deferred@1816132298 after retry RPC@238329635) -> (continuation of Deferred@1142118535 after retry RPC@1861491361))) after lookup
00:28:44.408 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:44.409 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.409 [DEBUG - kudu-nio-6] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.409 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.726 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:44.726 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:44.727 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=489), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [10ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [30ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [32ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [32ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [49ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [50ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [51ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [51ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [166ms] refreshing cache from master, [166ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [169ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [170ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [172ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [172ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [486ms] refreshing cache from master, [486ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [489ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@632231478(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@232bf413 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711) -> (continuation of Deferred@1816132298 after retry RPC@238329635) -> (continuation of Deferred@1142118535 after retry RPC@1861491361) -> (continuation of Deferred@1417123428 after retry RPC@446213977), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ecc3ab5 -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711) -> (continuation of Deferred@1816132298 after retry RPC@238329635) -> (continuation of Deferred@1142118535 after retry RPC@1861491361) -> (continuation of Deferred@1417123428 after retry RPC@446213977))) after lookup
00:28:44.727 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:44.728 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.728 [DEBUG - kudu-nio-7] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:44.728 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:45.086 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:45.087 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155),d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)[L],d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:45.087 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=849), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [10ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [30ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [32ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [32ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [49ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [50ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [51ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [51ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [166ms] refreshing cache from master, [166ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [169ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [170ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [172ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [172ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [486ms] refreshing cache from master, [486ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [489ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [490ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [491ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [491ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [846ms] refreshing cache from master, [846ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [849ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@632231478(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@232bf413 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711) -> (continuation of Deferred@1816132298 after retry RPC@238329635) -> (continuation of Deferred@1142118535 after retry RPC@1861491361) -> (continuation of Deferred@1417123428 after retry RPC@446213977) -> (continuation of Deferred@481542697 after retry RPC@1125008513), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ecc3ab5 -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711) -> (continuation of Deferred@1816132298 after retry RPC@238329635) -> (continuation of Deferred@1142118535 after retry RPC@1861491361) -> (continuation of Deferred@1417123428 after retry RPC@446213977) -> (continuation of Deferred@481542697 after retry RPC@1125008513))) after lookup
00:28:45.088 [DEBUG - kudu-nio-0] (Connection.java:748) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] connecting to peer
00:28:45.088 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:45.088 [DEBUG - kudu-nio-0] (Connection.java:709) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] cleaning up while in state CONNECTING due to: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:45.089 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133) for tablet 5209d2744dd54173958e67ef6c703429: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133
00:28:45.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.315631  3636 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 10 FOLLOWER]: Starting pre-election (detected failure of leader d0f7154f76e840bc92259656e7fd7d0b)
00:28:45.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.316031  3636 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:45.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.318239  3636 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:45.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:45.322217  3321 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:28:45.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:45.326243  3321 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:28:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.329053  3578 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "0f84a67fab33464bb93a32fd5b934b8d" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" is_pre_election: true
00:28:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.329689  3578 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f84a67fab33464bb93a32fd5b934b8d in term 9.
00:28:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.330924  3323 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: d0f7154f76e840bc92259656e7fd7d0b
00:28:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.331614  3636 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 10 FOLLOWER]: Leader pre-election won for term 11
00:28:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.331877  3636 raft_consensus.cc:493] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 10 FOLLOWER]: Starting leader election (detected failure of leader d0f7154f76e840bc92259656e7fd7d0b)
00:28:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.332180  3636 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 10 FOLLOWER]: Advancing to term 11
00:28:45.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.336535  3636 raft_consensus.cc:515] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:45.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.338057  3636 leader_election.cc:290] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 11 election: Requested vote from peers d7f44bee9e9544549e3b9aebe7adbdfe (127.1.37.3:36659), d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.338778  3578 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "5209d2744dd54173958e67ef6c703429" candidate_uuid: "0f84a67fab33464bb93a32fd5b934b8d" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe"
00:28:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.339229  3578 raft_consensus.cc:3060] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 9 FOLLOWER]: Advancing to term 11
00:28:45.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:45.342096  3321 leader_election.cc:336] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111)
00:28:45.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.346177  3578 raft_consensus.cc:2468] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f84a67fab33464bb93a32fd5b934b8d in term 11.
00:28:45.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.346964  3323 leader_election.cc:304] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f84a67fab33464bb93a32fd5b934b8d, d7f44bee9e9544549e3b9aebe7adbdfe; no voters: d0f7154f76e840bc92259656e7fd7d0b
00:28:45.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.347774  3636 raft_consensus.cc:2804] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 11 FOLLOWER]: Leader election won for term 11
00:28:45.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.349359  3636 raft_consensus.cc:697] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [term 11 LEADER]: Becoming Leader. State: Replica: 0f84a67fab33464bb93a32fd5b934b8d, State: Running, Role: LEADER
00:28:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.350325  3636 consensus_queue.cc:237] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:45.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.357735  1371 catalog_manager.cc:5696] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d reported cstate change: term changed from 10 to 11, leader changed from d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1) to 0f84a67fab33464bb93a32fd5b934b8d (127.1.37.2). New cstate: current_term: 11 leader_uuid: "0f84a67fab33464bb93a32fd5b934b8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } health_report { overall_health: UNKNOWN } } }
00:28:45.627 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 5209d2744dd54173958e67ef6c703429 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407' with partition [<start>, <end>)
00:28:45.627 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=5209d2744dd54173958e67ef6c703429@[0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)[L],d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133),d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)]}]
00:28:45.627 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=5209d2744dd54173958e67ef6c703429, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1390), Traces: [0ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [2ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [2ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [6ms] refreshing cache from master, [6ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [9ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [10ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [11ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [11ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [29ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [30ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [32ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [32ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [46ms] refreshing cache from master, [46ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [49ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [50ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [51ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [51ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [166ms] refreshing cache from master, [166ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [169ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [170ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [172ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [172ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [486ms] refreshing cache from master, [486ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [489ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [490ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [491ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [491ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [846ms] refreshing cache from master, [846ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [849ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, [850ms] sending RPC to server d0f7154f76e840bc92259656e7fd7d0b, [851ms] received response from server d0f7154f76e840bc92259656e7fd7d0b: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [852ms] delaying RPC due to: Network error: Failed to connect to peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133): Connection refused: /127.1.37.1:36133, [1386ms] refreshing cache from master, [1386ms] Sub RPC GetTableLocations: sending RPC to server master-127.1.37.60:42571, [1390ms] Sub RPC GetTableLocations: received response from server master-127.1.37.60:42571: OK, deferred=Deferred@632231478(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$229/0x00007fb7e436b330@232bf413 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711) -> (continuation of Deferred@1816132298 after retry RPC@238329635) -> (continuation of Deferred@1142118535 after retry RPC@1861491361) -> (continuation of Deferred@1417123428 after retry RPC@446213977) -> (continuation of Deferred@481542697 after retry RPC@1125008513) -> (continuation of Deferred@283276400 after retry RPC@937899098), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4ecc3ab5 -> wakeup thread Time-limited test -> (continuation of Deferred@2056289690 after retry RPC@1428617711) -> (continuation of Deferred@1816132298 after retry RPC@238329635) -> (continuation of Deferred@1142118535 after retry RPC@1861491361) -> (continuation of Deferred@1417123428 after retry RPC@446213977) -> (continuation of Deferred@481542697 after retry RPC@1125008513) -> (continuation of Deferred@283276400 after retry RPC@937899098))) after lookup
00:28:45.628 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] connecting to peer
00:28:45.629 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] Successfully connected to peer
00:28:45.649 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd3bb8cd5, L:/127.0.0.1:55632 - R:/127.1.37.2:37155] using TOKEN/null
00:28:45.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.659989  3578 raft_consensus.cc:1275] T 5209d2744dd54173958e67ef6c703429 P d7f44bee9e9544549e3b9aebe7adbdfe [term 11 FOLLOWER]: Refusing update from remote peer 0f84a67fab33464bb93a32fd5b934b8d: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 42. (index mismatch)
00:28:45.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:45.660816  3321 consensus_peers.cc:597] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d -> Peer d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133): Couldn't send request to peer d0f7154f76e840bc92259656e7fd7d0b. Status: Network error: Client connection negotiation failed: client connection to 127.1.37.1:36133: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:28:45.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.661720  3641 consensus_queue.cc:1048] T 5209d2744dd54173958e67ef6c703429 P 0f84a67fab33464bb93a32fd5b934b8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.001s
00:28:45.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.675992  3657 mvcc.cc:204] Tried to move back new op lower bound from 7305095257662263296 to 7305095249330237440. Current Snapshot: MvccSnapshot[applied={T|T < 7305095257662263296}]
00:28:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.684700  3655 mvcc.cc:204] Tried to move back new op lower bound from 7305095273083318272 to 7305095271842267136. Current Snapshot: MvccSnapshot[applied={T|T < 7305095273083318272}]
00:28:45.724 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.1.37.1:36133
00:28:45.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

00:28:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:45.726392  1172 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu
00:28:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data
00:28:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/logs
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.1:36133
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.1
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44645
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.1
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:46.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:46.023686  3664 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
00:28:46.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:46.024140  3664 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:46.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:46.024659  3664 flags.cc:432] Enabled unsafe flag: --never_fsync=true
00:28:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:46.056614  3664 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
00:28:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:46.057410  3664 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.37.1
00:28:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:46.096956  3664 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.37.20:43023
00:28:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:46.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.37.1:36133
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.37.1
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44645
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.37.62:40671,127.1.37.61:35087,127.1.37.60:42571
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.37.1
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/logs
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d8247991d193c4f984e0c0bdfb0389121fdc8d9
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 08 Jul 2026 00:13:11 UTC on e7f111948823
00:28:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 16843
00:28:46.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:46.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:46.098650  3664 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:46.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:46.100710  3664 file_cache.cc:492] Constructed file cache file cache with capacity 419430
00:28:46.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:46.116073  3670 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:46.117379  3671 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:47.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:47.445376  3673 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:47.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:47.445570  3664 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.329s	user 0.481s	sys 0.822s
00:28:47.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:47.445881  3664 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.330s	user 0.481s	sys 0.822s
00:28:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:47.449097  3672 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1326 milliseconds
00:28:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.449182  3664 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:47.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.451565  3664 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:47.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.454152  3664 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:47.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.455575  3664 hybrid_clock.cc:648] HybridClock initialized: now 1783470527455523 us; error 41 us; skew 500 ppm
00:28:47.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.456674  3664 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:47.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.465005  3664 webserver.cc:534] Webserver started at http://127.1.37.1:44645/ using document root <none> and password file <none>
00:28:47.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.466224  3664 fs_manager.cc:362] Metadata directory not provided
00:28:47.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.466499  3664 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:47.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.476830  3664 fs_manager.cc:714] Time spent opening directory manager: real 0.006s	user 0.006s	sys 0.002s
00:28:47.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.482488  3686 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
00:28:47.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.483647  3664 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.001s
00:28:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.483942  3664 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data,/tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0f7154f76e840bc92259656e7fd7d0b"
00:28:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-07-08 00:27:51 on dist-test-slave-4l7f"
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.485843  3664 fs_report.cc:389] FS layout report
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/wal
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/data
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:47.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.548555  3664 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
00:28:47.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.550040  3664 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:47.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.550487  3664 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.553045  3664 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
00:28:47.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.558635  3693 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
00:28:47.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.566448  3664 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
00:28:47.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.566659  3664 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.001s	sys 0.001s
00:28:47.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.566887  3664 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
00:28:47.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.571344  3664 ts_tablet_manager.cc:616] Registered 1 tablets
00:28:47.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.571517  3664 ts_tablet_manager.cc:595] Time spent register tablets: real 0.005s	user 0.002s	sys 0.000s
00:28:47.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.571920  3693 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap starting.
00:28:47.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.639350  3693 log.cc:826] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Log is configured to *not* fsync() on all Append() calls
00:28:47.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.760277  3693 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:47.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.760490  3664 rpc_server.cc:307] RPC server started. Bound to: 127.1.37.1:36133
00:28:47.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.761076  3802 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.37.1:36133 every 8 connection(s)
00:28:47.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.761246  3693 tablet_bootstrap.cc:492] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Bootstrap complete.
00:28:47.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.763140  3693 ts_tablet_manager.cc:1403] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Time spent bootstrapping tablet: real 0.192s	user 0.136s	sys 0.047s
00:28:47.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.763314  3664 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwJm9or/test-tmp/mini-kudu-cluster17356060921479103401/ts-0/data/info.pb
00:28:47.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.773386  1172 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu as pid 3664
00:28:47.774 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
00:28:47.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.785305  3693 raft_consensus.cc:359] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:47.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.788300  3693 raft_consensus.cc:740] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: d0f7154f76e840bc92259656e7fd7d0b, State: Initialized, Role: FOLLOWER
00:28:47.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.789830  3693 consensus_queue.cc:260] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d7f44bee9e9544549e3b9aebe7adbdfe" member_type: VOTER last_known_addr { host: "127.1.37.3" port: 36659 } } peers { permanent_uuid: "0f84a67fab33464bb93a32fd5b934b8d" member_type: VOTER last_known_addr { host: "127.1.37.2" port: 37155 } } peers { permanent_uuid: "d0f7154f76e840bc92259656e7fd7d0b" member_type: VOTER last_known_addr { host: "127.1.37.1" port: 36133 } }
00:28:47.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.799168  3693 ts_tablet_manager.cc:1434] T 5209d2744dd54173958e67ef6c703429 P d0f7154f76e840bc92259656e7fd7d0b: Time spent starting tablet: real 0.036s	user 0.028s	sys 0.008s
00:28:47.811 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:47.827 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1783470476407, tablet=5209d2744dd54173958e67ef6c703429, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
00:28:47.834 [DEBUG - main] (Connection.java:575) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] disconnecting while in state READY
00:28:47.835 [DEBUG - main] (Connection.java:575) [peer master-127.1.37.60:42571(127.1.37.60:42571)] disconnecting while in state READY
00:28:47.835 [DEBUG - main] (Connection.java:575) [peer d7f44bee9e9544549e3b9aebe7adbdfe(127.1.37.3:36659)] disconnecting while in state TERMINATED
00:28:47.836 [DEBUG - main] (Connection.java:575) [peer d0f7154f76e840bc92259656e7fd7d0b(127.1.37.1:36133)] disconnecting while in state TERMINATED
00:28:47.836 [DEBUG - main] (Connection.java:575) [peer master-127.1.37.62:40671(127.1.37.62:40671)] disconnecting while in state READY
00:28:47.836 [DEBUG - main] (Connection.java:575) [peer master-127.1.37.61:35087(127.1.37.61:35087)] disconnecting while in state READY
00:28:47.839 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.1.37.61:35087(127.1.37.61:35087)] handling channelInactive
00:28:47.839 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.1.37.61:35087(127.1.37.61:35087)] cleaning up while in state READY due to: connection closed
00:28:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.838629  3803 heartbeater.cc:344] Connected to a master server at 127.1.37.60:42571
00:28:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.839160  3803 heartbeater.cc:461] Registering TS with master...
00:28:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.840581  3803 heartbeater.cc:507] Master 127.1.37.60:42571 requested a full tablet report, sending...
00:28:47.841 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.37.60:42571(127.1.37.60:42571)] handling channelInactive
00:28:47.841 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.37.60:42571(127.1.37.60:42571)] cleaning up while in state READY due to: connection closed
00:28:47.843 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:28:47.848 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] handling channelInactive
00:28:47.848 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 0f84a67fab33464bb93a32fd5b934b8d(127.1.37.2:37155)] cleaning up while in state READY due to: connection closed
00:28:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.845840  1372 ts_manager.cc:194] Re-registered known tserver with Master: d0f7154f76e840bc92259656e7fd7d0b (127.1.37.1:36133)
00:28:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.848404  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 3664
00:28:47.849 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.37.62:40671(127.1.37.62:40671)] handling channelInactive
00:28:47.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.37.62:40671(127.1.37.62:40671)] cleaning up while in state READY due to: connection closed
00:28:47.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.853330  1372 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.37.1:44535
00:28:47.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:47.876091  1358 connection.cc:573] server connection from 127.1.37.1:44535 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:28:47.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.876516  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 3300
00:28:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:47.877781  3811 negotiation.cc:336] Failed RPC negotiation. Trace:
00:28:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.818171 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.1.37.1:60081 (local address 127.1.37.61:35087)
00:28:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.824041 (+  5870us) server_negotiation.cc:207] Beginning negotiation
00:28:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.824076 (+    35us) server_negotiation.cc:400] Waiting for connection header
00:28:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.824132 (+    56us) server_negotiation.cc:408] Connection header received
00:28:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.824324 (+   192us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.824351 (+    27us) server_negotiation.cc:462] Received NEGOTIATE request from client
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.824610 (+   259us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.838821 (+ 14211us) server_negotiation.cc:227] Negotiated authn=SASL
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.876237 (+ 37416us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.37.1:60081: BlockingRecv error: recv got EOF from 127.1.37.1:60081 (error 108)
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5180,"thread_start_us":4882,"threads_started":1}
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260708 00:28:47.878865  3813 negotiation.cc:336] Failed RPC negotiation. Trace:
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.828455 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.1.37.1:34157 (local address 127.1.37.62:40671)
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.857536 (+ 29081us) server_negotiation.cc:207] Beginning negotiation
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.857561 (+    25us) server_negotiation.cc:400] Waiting for connection header
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.857608 (+    47us) server_negotiation.cc:408] Connection header received
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.857737 (+   129us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.857760 (+    23us) server_negotiation.cc:462] Received NEGOTIATE request from client
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.857984 (+   224us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.858148 (+   164us) server_negotiation.cc:227] Negotiated authn=SASL
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0708 00:28:47.877432 (+ 19284us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.37.1:34157: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
00:28:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":28550,"thread_start_us":28266,"threads_started":1}
00:28:47.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.901934  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 3486
00:28:47.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.924952  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 1184
00:28:47.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.951327  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 1254
00:28:47.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260708 00:28:47.975907  1172 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwJm9or/build/tsan/bin/kudu with pid 1329
00:28:49.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-07-08T00:28:49Z chronyd exiting

Time: 68.528

OK (1 test)

