JUnit version 4.13.2
.18:20:45.496 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1786472445488
18:20:45.550 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
18:20:45.582 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
18:20:45.593 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskiKkpA8/test-tmp
18:20:45.619 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
18:20:45.621 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskiKkpA8/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
18:20:45.838 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

18:20:46.084 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
18:20:46.085 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

18:20:46.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.089529 20485 mini_oidc.cc:150] Starting JWKS server
18:20:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.092947 20485 webserver.cc:533] Webserver started at http://127.0.0.1:40377/ using document root <none> and password file <none>
18:20:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.093278 20485 mini_oidc.cc:186] Starting OIDC Discovery server
18:20:46.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.094091 20485 webserver.cc:533] Webserver started at http://127.0.0.1:42041/ using document root <none> and password file <none>
18:20:46.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T18:20:46Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
18:20:46.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T18:20:46Z Disabled control of system clock
18:20:46.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.134179 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:46.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:46.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/wal
18:20:46.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/data
18:20:46.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:20:46.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:20:46.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:20:46.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:46.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:46.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:46.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/logs
18:20:46.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/data/info.pb
18:20:46.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:46.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:20:46.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:20:46.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:20:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:20:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:20:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:20:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:20:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:20:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.126:36795
18:20:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.126
18:20:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:20:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:20:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40377/jwks.json with env {}
18:20:46.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.507382 20498 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:20:46.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.507742 20498 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:20:46.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.507823 20498 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:20:46.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.516498 20498 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
18:20:46.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.516609 20498 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:20:46.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.516676 20498 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
18:20:46.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.516781 20498 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
18:20:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.516846 20498 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
18:20:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.516882 20498 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:40377/jwks.json
18:20:46.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.527462 20498 master_runner.cc:387] Master server non-default flags:
18:20:46.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/logs
18:20:46.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:46.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:20:46.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/data
18:20:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/wal
18:20:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:20:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:20:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.126:36795
18:20:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:20:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:20:46.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:20:46.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40377/jwks.json
18:20:46.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:46.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:46.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/data/info.pb
18:20:46.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.126
18:20:46.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:46.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:20:46.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:20:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:20:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:46.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:20:46.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:20:46.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:20:46.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:20:46.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:20:46.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:20:46.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:20:46.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:20:46.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.529567 20498 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:20:46.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.531843 20498 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:20:46.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.542146 20503 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.542127 20508 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.542338 20505 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.543125 20498 server_base.cc:1061] running on GCE node
18:20:46.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.544516 20498 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:20:46.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.546021 20498 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:20:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.547297 20498 hybrid_clock.cc:648] HybridClock initialized: now 1786472446547183 us; error 167 us; skew 500 ppm
18:20:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.547727 20498 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:20:46.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.550081 20498 webserver.cc:533] Webserver started at http://127.20.1.126:33441/ using document root <none> and password file <none>
18:20:46.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.550856 20498 fs_manager.cc:362] Metadata directory not provided
18:20:46.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.550952 20498 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:20:46.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.551379 20498 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:20:46.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.554024 20498 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/data/instance:
18:20:46.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "707b21376dce40cca0ce20bcd5cd777f"
18:20:46.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:46 on dist-test-slave-6k22"
18:20:46.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.554943 20498 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/wal/instance:
18:20:46.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "707b21376dce40cca0ce20bcd5cd777f"
18:20:46.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:46 on dist-test-slave-6k22"
18:20:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.561154 20498 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.003s
18:20:46.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.565080 20516 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:20:46.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.566708 20498 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
18:20:46.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.566859 20498 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/wal
18:20:46.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "707b21376dce40cca0ce20bcd5cd777f"
18:20:46.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:46 on dist-test-slave-6k22"
18:20:46.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.567111 20498 fs_report.cc:389] FS layout report
18:20:46.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:20:46.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/wal
18:20:46.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/wal
18:20:46.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/data/data
18:20:46.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:20:46.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:20:46.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:20:46.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:20:46.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:20:46.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:20:46.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:20:46.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:20:46.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:20:46.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:20:46.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:20:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.589473 20498 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:20:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.590391 20498 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:20:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.590665 20498 kserver.cc:163] Server-wide thread pool size limit: 3276
18:20:46.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.613617 20498 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.126:36795
18:20:46.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.613643 20572 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.126:36795 every 8 connection(s)
18:20:46.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.615525 20498 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/data/info.pb
18:20:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.620618 20574 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:20:46.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.626186 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 20498
18:20:46.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.626559 20485 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-0/wal/instance
18:20:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.627393 20574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } has no permanent_uuid. Determining permanent_uuid...
18:20:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.629837 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/wal
18:20:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/data
18:20:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:20:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:20:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:20:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:46.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/logs
18:20:46.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/data/info.pb
18:20:46.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:46.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:20:46.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:20:46.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:20:46.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:20:46.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:46.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:46.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:46.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:20:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:20:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:20:46.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:20:46.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.125:35653
18:20:46.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.125
18:20:46.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:46.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:46.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:46.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:46.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:46.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:20:46.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:46.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:20:46.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40377/jwks.json with env {}
18:20:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.657948 20574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } has no permanent_uuid. Determining permanent_uuid...
18:20:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.659216 20519 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.125:35653: connect: Connection refused (error 111)
18:20:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.661026 20574 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.125:35653: Network error: Client connection negotiation failed: client connection to 127.20.1.125:35653: connect: Connection refused (error 111)
18:20:46.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.710384 20574 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } attempt: 1
18:20:46.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.712970 20574 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.125:35653: Network error: Client connection negotiation failed: client connection to 127.20.1.125:35653: connect: Connection refused (error 111)
18:20:46.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.781280 20574 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } attempt: 2
18:20:46.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.783764 20574 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.125:35653: Network error: Client connection negotiation failed: client connection to 127.20.1.125:35653: connect: Connection refused (error 111)
18:20:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:46.875029 20574 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } attempt: 3
18:20:46.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:46.877759 20574 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.125:35653: Network error: Client connection negotiation failed: client connection to 127.20.1.125:35653: connect: Connection refused (error 111)
18:20:47.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.013144 20578 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:20:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.013504 20578 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:20:47.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.013584 20578 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:20:47.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.021085 20574 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } attempt: 4
18:20:47.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.023047 20578 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
18:20:47.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.023141 20578 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:20:47.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.023207 20578 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
18:20:47.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.023249 20578 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
18:20:47.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.023375 20578 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
18:20:47.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.023432 20578 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:40377/jwks.json
18:20:47.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.023936 20574 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.125:35653: Network error: Client connection negotiation failed: client connection to 127.20.1.125:35653: connect: Connection refused (error 111)
18:20:47.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.040309 20578 master_runner.cc:387] Master server non-default flags:
18:20:47.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/logs
18:20:47.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:47.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:20:47.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:47.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/data
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/wal
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.125:35653
18:20:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:20:47.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:20:47.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:20:47.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40377/jwks.json
18:20:47.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:47.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:47.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/data/info.pb
18:20:47.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.125
18:20:47.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:47.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:20:47.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:47.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:20:47.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:20:47.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:20:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:20:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:20:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:20:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:20:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:20:47.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:20:47.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:20:47.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.042686 20578 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:20:47.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.045285 20578 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:20:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.055429 20585 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:47.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.056684 20584 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:47.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.057824 20587 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:47.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.058172 20578 server_base.cc:1061] running on GCE node
18:20:47.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.059337 20578 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:20:47.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.061203 20578 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:20:47.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.062453 20578 hybrid_clock.cc:648] HybridClock initialized: now 1786472447062387 us; error 60 us; skew 500 ppm
18:20:47.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.062855 20578 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:20:47.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.065308 20578 webserver.cc:533] Webserver started at http://127.20.1.125:38031/ using document root <none> and password file <none>
18:20:47.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.066156 20578 fs_manager.cc:362] Metadata directory not provided
18:20:47.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.066252 20578 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:20:47.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.066702 20578 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:20:47.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.069322 20578 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/data/instance:
18:20:47.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c5d1edea27f4ad493c9e4720c3ec22e"
18:20:47.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:47 on dist-test-slave-6k22"
18:20:47.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.070066 20578 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/wal/instance:
18:20:47.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c5d1edea27f4ad493c9e4720c3ec22e"
18:20:47.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:47 on dist-test-slave-6k22"
18:20:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.075987 20578 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.000s
18:20:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.079705 20594 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:20:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.081182 20578 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
18:20:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.081410 20578 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/wal
18:20:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c5d1edea27f4ad493c9e4720c3ec22e"
18:20:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:47 on dist-test-slave-6k22"
18:20:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.081671 20578 fs_report.cc:389] FS layout report
18:20:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:20:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/wal
18:20:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/wal
18:20:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/data/data
18:20:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:20:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:20:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:20:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:20:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:20:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:20:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:20:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:20:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:20:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:20:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:20:47.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.117790 20578 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:20:47.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.119012 20578 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:20:47.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.119383 20578 kserver.cc:163] Server-wide thread pool size limit: 3276
18:20:47.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.142729 20578 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.125:35653
18:20:47.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.142753 20651 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.125:35653 every 8 connection(s)
18:20:47.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.144706 20578 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/data/info.pb
18:20:47.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.148466 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 20578
18:20:47.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.148607 20485 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-1/wal/instance
18:20:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.149788 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/wal
18:20:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/data
18:20:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:20:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:20:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:20:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/logs
18:20:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/data/info.pb
18:20:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:47.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:20:47.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:20:47.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:20:47.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:20:47.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:47.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:47.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:20:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:20:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:20:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:20:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.124:35869
18:20:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.124
18:20:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:20:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:20:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40377/jwks.json with env {}
18:20:47.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.149950 20652 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:20:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.158780 20652 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } has no permanent_uuid. Determining permanent_uuid...
18:20:47.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.183034 20652 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } has no permanent_uuid. Determining permanent_uuid...
18:20:47.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.193998 20652 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } has no permanent_uuid. Determining permanent_uuid...
18:20:47.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.195338 20598 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.124:35869: connect: Connection refused (error 111)
18:20:47.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.197395 20652 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.124:35869: Network error: Client connection negotiation failed: client connection to 127.20.1.124:35869: connect: Connection refused (error 111)
18:20:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.246801 20652 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } attempt: 1
18:20:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.249213 20652 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.124:35869: Network error: Client connection negotiation failed: client connection to 127.20.1.124:35869: connect: Connection refused (error 111)
18:20:47.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.317533 20652 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } attempt: 2
18:20:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.320211 20652 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.124:35869: Network error: Client connection negotiation failed: client connection to 127.20.1.124:35869: connect: Connection refused (error 111)
18:20:47.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.323201 20574 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } attempt: 5
18:20:47.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.330305 20574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } has no permanent_uuid. Determining permanent_uuid...
18:20:47.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.332391 20574 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.124:35869: Network error: Client connection negotiation failed: client connection to 127.20.1.124:35869: connect: Connection refused (error 111)
18:20:47.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.383761 20574 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } attempt: 1
18:20:47.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.386056 20574 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.124:35869: Network error: Client connection negotiation failed: client connection to 127.20.1.124:35869: connect: Connection refused (error 111)
18:20:47.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.411548 20652 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } attempt: 3
18:20:47.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.414916 20652 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.124:35869: Network error: Client connection negotiation failed: client connection to 127.20.1.124:35869: connect: Connection refused (error 111)
18:20:47.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.454373 20574 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } attempt: 2
18:20:47.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.456719 20574 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.124:35869: Network error: Client connection negotiation failed: client connection to 127.20.1.124:35869: connect: Connection refused (error 111)
18:20:47.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.529176 20654 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:20:47.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.529511 20654 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:20:47.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.529588 20654 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:20:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.538163 20654 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
18:20:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.538264 20654 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:20:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.538444 20654 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
18:20:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.538493 20654 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
18:20:47.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.538551 20654 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
18:20:47.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.538587 20654 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:40377/jwks.json
18:20:47.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.552392 20654 master_runner.cc:387] Master server non-default flags:
18:20:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/logs
18:20:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:20:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/data
18:20:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/wal
18:20:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:20:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:20:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.124:35869
18:20:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:20:47.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:20:47.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:20:47.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40377/jwks.json
18:20:47.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:47.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:47.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/data/info.pb
18:20:47.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.124
18:20:47.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:47.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:20:47.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:47.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:20:47.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:20:47.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:47.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:20:47.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:20:47.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:20:47.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:20:47.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:20:47.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:20:47.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:20:47.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:20:47.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.554338 20654 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:20:47.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.556742 20654 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:20:47.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.558182 20652 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } attempt: 4
18:20:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.560568 20652 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.124:35869: Network error: Client connection negotiation failed: client connection to 127.20.1.124:35869: connect: Connection refused (error 111)
18:20:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.562997 20574 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } attempt: 3
18:20:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.565495 20574 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.1.124:35869: Network error: Client connection negotiation failed: client connection to 127.20.1.124:35869: connect: Connection refused (error 111)
18:20:47.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.567226 20666 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:47.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.567255 20665 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:47.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.568522 20669 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:47.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.569001 20654 server_base.cc:1061] running on GCE node
18:20:47.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.569967 20654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:20:47.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.571641 20654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:20:47.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.572914 20654 hybrid_clock.cc:648] HybridClock initialized: now 1786472447572841 us; error 61 us; skew 500 ppm
18:20:47.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.573388 20654 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:20:47.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.575994 20654 webserver.cc:533] Webserver started at http://127.20.1.124:40603/ using document root <none> and password file <none>
18:20:47.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.576937 20654 fs_manager.cc:362] Metadata directory not provided
18:20:47.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.577083 20654 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:20:47.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.577589 20654 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:20:47.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.580533 20654 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/data/instance:
18:20:47.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811"
18:20:47.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:47 on dist-test-slave-6k22"
18:20:47.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.581557 20654 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/wal/instance:
18:20:47.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811"
18:20:47.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:47 on dist-test-slave-6k22"
18:20:47.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.587740 20654 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.001s
18:20:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.591852 20679 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:20:47.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.593374 20654 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
18:20:47.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.593809 20654 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/wal
18:20:47.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811"
18:20:47.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:47 on dist-test-slave-6k22"
18:20:47.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.594094 20654 fs_report.cc:389] FS layout report
18:20:47.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:20:47.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/wal
18:20:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/wal
18:20:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/data/data
18:20:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:20:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:20:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:20:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:20:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:20:47.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:20:47.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:20:47.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:20:47.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:20:47.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:20:47.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:20:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.630781 20654 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:20:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.631963 20654 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:20:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.632326 20654 kserver.cc:163] Server-wide thread pool size limit: 3276
18:20:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.656020 20740 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.124:35869 every 8 connection(s)
18:20:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.656044 20654 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.124:35869
18:20:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.657855 20654 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/data/info.pb
18:20:47.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.662453 20741 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:20:47.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.665063 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 20654
18:20:47.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.665282 20485 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/master-2/wal/instance
18:20:47.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.670306 20741 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } has no permanent_uuid. Determining permanent_uuid...
18:20:47.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.699229 20741 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } has no permanent_uuid. Determining permanent_uuid...
18:20:47.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.708122 20741 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } has no permanent_uuid. Determining permanent_uuid...
18:20:47.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.720016 20741 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811: Bootstrap starting.
18:20:47.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.723798 20741 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811: Neither blocks nor log segments found. Creating new log.
18:20:47.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.726046 20741 log.cc:826] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811: Log is configured to *not* fsync() on all Append() calls
18:20:47.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.729470 20741 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811: No bootstrap required, opened a new log
18:20:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.735515 20741 raft_consensus.cc:359] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } }
18:20:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.735852 20741 raft_consensus.cc:385] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:20:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.735951 20741 raft_consensus.cc:740] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cc0a8e2e771b46b78b5b84dd4b5b9811, State: Initialized, Role: FOLLOWER
18:20:47.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.736685 20741 consensus_queue.cc:260] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } }
18:20:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.737926 20749 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } } }
18:20:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.738337 20749 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [sys.catalog]: This master's current role is: FOLLOWER
18:20:47.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.739276 20741 sys_catalog.cc:565] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [sys.catalog]: configured and running, proceeding with master startup.
18:20:47.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.742822 20574 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } attempt: 4
18:20:47.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.755149 20763 catalog_manager.cc:1594] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:20:47.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.755295 20763 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:20:47.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.757375 20574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f: Bootstrap starting.
18:20:47.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.752658 20654 master_runner.cc:187] Error getting master registration for 127.20.1.126:36795: OK, instance_id { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" instance_seqno: 1786472446607610 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:20:47.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.758190 20654 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
18:20:47.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.761073 20574 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f: Neither blocks nor log segments found. Creating new log.
18:20:47.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.762247 20574 log.cc:826] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f: Log is configured to *not* fsync() on all Append() calls
18:20:47.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.766040 20574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f: No bootstrap required, opened a new log
18:20:47.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.774971 20574 raft_consensus.cc:359] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } }
18:20:47.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.775347 20574 raft_consensus.cc:385] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:20:47.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.775481 20574 raft_consensus.cc:740] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 707b21376dce40cca0ce20bcd5cd777f, State: Initialized, Role: FOLLOWER
18:20:47.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.776365 20574 consensus_queue.cc:260] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } }
18:20:47.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.777707 20766 sys_catalog.cc:455] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } } }
18:20:47.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.778064 20766 sys_catalog.cc:458] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [sys.catalog]: This master's current role is: FOLLOWER
18:20:47.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.779163 20574 sys_catalog.cc:565] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [sys.catalog]: configured and running, proceeding with master startup.
18:20:47.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.794914 20780 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:20:47.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.795872 20780 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:20:47.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.793412 20498 master_runner.cc:187] Error getting master registration for 127.20.1.125:35653: OK, instance_id { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" instance_seqno: 1786472447136485 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:20:47.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.806429 20498 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
18:20:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.844054 20523 tablet.cc:2406] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:20:47.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.859884 20652 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } attempt: 5
18:20:47.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.877254 20652 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e: Bootstrap starting.
18:20:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.883378 20652 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e: Neither blocks nor log segments found. Creating new log.
18:20:47.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.884898 20685 tablet.cc:2406] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:20:47.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.886611 20652 log.cc:826] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e: Log is configured to *not* fsync() on all Append() calls
18:20:47.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.890910 20652 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e: No bootstrap required, opened a new log
18:20:47.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.898837 20652 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } }
18:20:47.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.899179 20652 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:20:47.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.899396 20652 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c5d1edea27f4ad493c9e4720c3ec22e, State: Initialized, Role: FOLLOWER
18:20:47.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.900635 20652 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } }
18:20:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.905926 20783 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } } }
18:20:47.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.906594 20783 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [sys.catalog]: This master's current role is: FOLLOWER
18:20:47.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.907500 20652 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [sys.catalog]: configured and running, proceeding with master startup.
18:20:47.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.930438 20578 master_runner.cc:264] No leader master found from master 9c5d1edea27f4ad493c9e4720c3ec22e
18:20:47.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:47.930639 20578 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
18:20:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.936517 20796 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:20:47.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:47.937156 20796 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:20:48.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.016537 20749 raft_consensus.cc:493] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:20:48.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.016949 20749 raft_consensus.cc:515] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } }
18:20:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.025782 20547 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "707b21376dce40cca0ce20bcd5cd777f" is_pre_election: true
18:20:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.026602 20547 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cc0a8e2e771b46b78b5b84dd4b5b9811 in term 0.
18:20:48.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.027522 20680 leader_election.cc:304] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 707b21376dce40cca0ce20bcd5cd777f, cc0a8e2e771b46b78b5b84dd4b5b9811; no voters: 
18:20:48.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.031994 20624 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" is_pre_election: true
18:20:48.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.032688 20624 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cc0a8e2e771b46b78b5b84dd4b5b9811 in term 0.
18:20:48.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.040241 20749 leader_election.cc:290] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 707b21376dce40cca0ce20bcd5cd777f (127.20.1.126:36795), 9c5d1edea27f4ad493c9e4720c3ec22e (127.20.1.125:35653)
18:20:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.040550 20749 raft_consensus.cc:2804] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 0 FOLLOWER]: Leader pre-election won for term 1
18:20:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.041285 20749 raft_consensus.cc:493] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:20:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.041420 20749 raft_consensus.cc:3060] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 0 FOLLOWER]: Advancing to term 1
18:20:48.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.057014 20749 raft_consensus.cc:515] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } }
18:20:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.065244 20624 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e"
18:20:48.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.065585 20624 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [term 0 FOLLOWER]: Advancing to term 1
18:20:48.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.069636 20624 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cc0a8e2e771b46b78b5b84dd4b5b9811 in term 1.
18:20:48.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.070274 20682 leader_election.cc:304] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c5d1edea27f4ad493c9e4720c3ec22e, cc0a8e2e771b46b78b5b84dd4b5b9811; no voters: 
18:20:48.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.072446 20749 leader_election.cc:290] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [CANDIDATE]: Term 1 election: Requested vote from peers 707b21376dce40cca0ce20bcd5cd777f (127.20.1.126:36795), 9c5d1edea27f4ad493c9e4720c3ec22e (127.20.1.125:35653)
18:20:48.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.072762 20749 raft_consensus.cc:2804] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 1 FOLLOWER]: Leader election won for term 1
18:20:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.072917 20749 raft_consensus.cc:697] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [term 1 LEADER]: Becoming Leader. State: Replica: cc0a8e2e771b46b78b5b84dd4b5b9811, State: Running, Role: LEADER
18:20:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.075582 20749 consensus_queue.cc:237] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } }
18:20:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:48.078068 20680 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.20.1.126:36795, user_credentials={real_user=slave}} blocked reactor thread for 50696.8us
18:20:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.078974 20547 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "707b21376dce40cca0ce20bcd5cd777f"
18:20:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.079236 20547 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [term 0 FOLLOWER]: Advancing to term 1
18:20:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.085963 20547 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cc0a8e2e771b46b78b5b84dd4b5b9811 in term 1.
18:20:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.116618 20798 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cc0a8e2e771b46b78b5b84dd4b5b9811. Latest consensus state: current_term: 1 leader_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } } }
18:20:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.117082 20798 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [sys.catalog]: This master's current role is: LEADER
18:20:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.119076 20800 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
18:20:48.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:48.122535 20601 tablet.cc:2406] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:20:48.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.131704 20800 catalog_manager.cc:1520] Initializing Kudu cluster ID...
18:20:48.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.188915 20547 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [term 1 FOLLOWER]: Refusing update from remote peer cc0a8e2e771b46b78b5b84dd4b5b9811: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:20:48.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.189062 20624 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [term 1 FOLLOWER]: Refusing update from remote peer cc0a8e2e771b46b78b5b84dd4b5b9811: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:20:48.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.190464 20798 consensus_queue.cc:1048] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [LEADER]: Connected to new peer: Peer: permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:20:48.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.191136 20749 consensus_queue.cc:1048] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:20:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.217118 20766 sys_catalog.cc:455] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [sys.catalog]: SysCatalogTable state changed. Reason: New leader cc0a8e2e771b46b78b5b84dd4b5b9811. Latest consensus state: current_term: 1 leader_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } } }
18:20:48.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.217484 20766 sys_catalog.cc:458] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [sys.catalog]: This master's current role is: FOLLOWER
18:20:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.222594 20783 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [sys.catalog]: SysCatalogTable state changed. Reason: New leader cc0a8e2e771b46b78b5b84dd4b5b9811. Latest consensus state: current_term: 1 leader_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } } }
18:20:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.222956 20783 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [sys.catalog]: This master's current role is: FOLLOWER
18:20:48.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.228394 20766 sys_catalog.cc:455] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } } }
18:20:48.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.228708 20766 sys_catalog.cc:458] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f [sys.catalog]: This master's current role is: FOLLOWER
18:20:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.232098 20798 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } } }
18:20:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.232347 20798 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [sys.catalog]: This master's current role is: LEADER
18:20:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.239498 20798 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } } }
18:20:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.239822 20798 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811 [sys.catalog]: This master's current role is: LEADER
18:20:48.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.243811 20800 catalog_manager.cc:1383] Generated new cluster ID: c8c053479a694456bdf7f3232cef91fe
18:20:48.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.243937 20800 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
18:20:48.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.244773 20783 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "707b21376dce40cca0ce20bcd5cd777f" member_type: VOTER last_known_addr { host: "127.20.1.126" port: 36795 } } peers { permanent_uuid: "9c5d1edea27f4ad493c9e4720c3ec22e" member_type: VOTER last_known_addr { host: "127.20.1.125" port: 35653 } } peers { permanent_uuid: "cc0a8e2e771b46b78b5b84dd4b5b9811" member_type: VOTER last_known_addr { host: "127.20.1.124" port: 35869 } } }
18:20:48.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.245074 20783 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e [sys.catalog]: This master's current role is: FOLLOWER
18:20:48.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.282819 20800 catalog_manager.cc:1406] Generated new certificate authority record
18:20:48.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.285372 20800 catalog_manager.cc:1540] Loading token signing keys...
18:20:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.304636 20800 catalog_manager.cc:6092] T 00000000000000000000000000000000 P cc0a8e2e771b46b78b5b84dd4b5b9811: Generated new TSK 0
18:20:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.306533 20800 catalog_manager.cc:1550] Initializing in-progress tserver states...
18:20:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.321664 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:20:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data
18:20:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:20:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:20:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:20:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/logs
18:20:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:20:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:20:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:20:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:20:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:20:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.65:0
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.65
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.65
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:20:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.798914 20780 catalog_manager.cc:1295] Loaded cluster ID: c8c053479a694456bdf7f3232cef91fe
18:20:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.799052 20780 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f: loading cluster ID for follower catalog manager: success
18:20:48.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.804731 20780 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f: acquiring CA information for follower catalog manager: success
18:20:48.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.808353 20780 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 707b21376dce40cca0ce20bcd5cd777f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:20:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.939872 20796 catalog_manager.cc:1295] Loaded cluster ID: c8c053479a694456bdf7f3232cef91fe
18:20:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.940012 20796 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e: loading cluster ID for follower catalog manager: success
18:20:48.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.943888 20796 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e: acquiring CA information for follower catalog manager: success
18:20:48.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:48.946509 20796 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 9c5d1edea27f4ad493c9e4720c3ec22e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:20:49.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.025190 20811 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:20:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.025801 20811 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:20:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.026010 20811 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:20:49.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.036412 20811 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:20:49.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.036604 20811 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.65
18:20:49.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.049372 20811 tablet_server_runner.cc:78] Tablet server non-default flags:
18:20:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/logs
18:20:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:20:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data
18:20:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:20:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.65:0
18:20:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:20:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:20:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.65
18:20:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:20:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:20:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:20:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.65
18:20:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:20:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:20:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:20:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:20:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:20:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:20:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:20:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:20:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.051383 20811 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:20:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.053574 20811 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:20:49.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.065810 20818 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:49.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.066345 20819 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.073501 20821 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.074357 20811 server_base.cc:1061] running on GCE node
18:20:49.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.075279 20811 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:20:49.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.076745 20811 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:20:49.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.078022 20811 hybrid_clock.cc:648] HybridClock initialized: now 1786472449077992 us; error 120 us; skew 500 ppm
18:20:49.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.078495 20811 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:20:49.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.081835 20811 webserver.cc:533] Webserver started at http://127.20.1.65:46555/ using document root <none> and password file <none>
18:20:49.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.082952 20811 fs_manager.cc:362] Metadata directory not provided
18:20:49.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.083084 20811 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:20:49.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.083424 20811 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:20:49.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.086400 20811 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/instance:
18:20:49.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90f4362b094e4d1681153e2008083f4e"
18:20:49.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:20:49.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.087412 20811 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal/instance:
18:20:49.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90f4362b094e4d1681153e2008083f4e"
18:20:49.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:20:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.094029 20811 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.005s	sys 0.000s
18:20:49.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.098263 20827 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.099840 20811 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.100008 20811 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90f4362b094e4d1681153e2008083f4e"
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.100194 20811 fs_report.cc:389] FS layout report
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/data
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:20:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:20:49.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:20:49.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:20:49.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:20:49.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:20:49.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:20:49.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:20:49.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:20:49.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.122612 20811 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:20:49.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.123792 20811 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:20:49.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.124497 20811 kserver.cc:163] Server-wide thread pool size limit: 3276
18:20:49.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.125954 20811 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:20:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.128198 20811 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:20:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.128332 20811 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:20:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.128432 20811 ts_tablet_manager.cc:616] Registered 0 tablets
18:20:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.128515 20811 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:20:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.195140 20811 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.65:36397
18:20:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.195261 20963 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.65:36397 every 8 connection(s)
18:20:49.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.196991 20811 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:20:49.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.205128 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 20811
18:20:49.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.205410 20485 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal/instance
18:20:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.218498 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:20:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data
18:20:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:20:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:20:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:20:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/logs
18:20:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:20:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:20:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:20:49.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:20:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:20:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:20:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:20:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.66:0
18:20:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.66
18:20:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.66
18:20:49.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:20:49.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.233741 20966 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:20:49.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.233884 20964 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:20:49.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.234289 20966 heartbeater.cc:461] Registering TS with master...
18:20:49.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.234290 20964 heartbeater.cc:461] Registering TS with master...
18:20:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.235940 20964 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:20:49.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.238155 20966 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:20:49.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.239120 20700 ts_manager.cc:194] Registered new tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:49.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.240512 20535 ts_manager.cc:194] Registered new tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:49.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.242208 20700 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.65:53003
18:20:49.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.245021 20967 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:20:49.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.245224 20967 heartbeater.cc:461] Registering TS with master...
18:20:49.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.245909 20967 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:20:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.247983 20612 ts_manager.cc:194] Registered new tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.756321 20975 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:20:49.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.756816 20975 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:20:49.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.756961 20975 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:20:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.769086 20975 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:20:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.769382 20975 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.66
18:20:49.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.782088 20975 tablet_server_runner.cc:78] Tablet server non-default flags:
18:20:49.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/logs
18:20:49.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:49.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:20:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data
18:20:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:20:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.66:0
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.66
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:49.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.66
18:20:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:20:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:20:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:20:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:20:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:20:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:20:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:20:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:20:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.784775 20975 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:20:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.787467 20975 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:20:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.802480 20981 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.803380 20984 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:49.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:49.804265 20982 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:49.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.805083 20975 server_base.cc:1061] running on GCE node
18:20:49.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.805733 20975 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:20:49.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.806851 20975 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:20:49.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.808166 20975 hybrid_clock.cc:648] HybridClock initialized: now 1786472449808016 us; error 74 us; skew 500 ppm
18:20:49.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.808589 20975 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:20:49.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.811663 20975 webserver.cc:533] Webserver started at http://127.20.1.66:40147/ using document root <none> and password file <none>
18:20:49.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.812706 20975 fs_manager.cc:362] Metadata directory not provided
18:20:49.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.812821 20975 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:20:49.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.813396 20975 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:20:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.816367 20975 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/instance:
18:20:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afe5bf8ec6e44b15bad84f94f3ee7158"
18:20:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:20:49.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.817217 20975 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal/instance:
18:20:49.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afe5bf8ec6e44b15bad84f94f3ee7158"
18:20:49.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:20:49.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.823885 20975 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.003s	sys 0.004s
18:20:49.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.828238 20991 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.829828 20975 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.830075 20975 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afe5bf8ec6e44b15bad84f94f3ee7158"
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.830431 20975 fs_report.cc:389] FS layout report
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/data
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:20:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:20:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:20:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:20:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:20:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:20:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:20:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:20:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:20:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:20:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.866349 20975 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:20:49.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.867337 20975 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:20:49.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.867811 20975 kserver.cc:163] Server-wide thread pool size limit: 3276
18:20:49.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.868949 20975 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:20:49.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.871095 20975 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:20:49.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.871222 20975 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:20:49.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.871330 20975 ts_tablet_manager.cc:616] Registered 0 tablets
18:20:49.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.871471 20975 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:20:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.937973 20975 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.66:40881
18:20:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.938230 21123 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.66:40881 every 8 connection(s)
18:20:49.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.939968 20975 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:20:49.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.944900 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 20975
18:20:49.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.945161 20485 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal/instance
18:20:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.951351 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:49.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/logs
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:20:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.67:0
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.67
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.67
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:20:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.976262 21126 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:20:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.977219 21125 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:20:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.977501 21126 heartbeater.cc:461] Registering TS with master...
18:20:49.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.977521 21125 heartbeater.cc:461] Registering TS with master...
18:20:49.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.979446 21126 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:20:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.979591 21125 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:20:49.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.982371 20612 ts_manager.cc:194] Registered new tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:20:49.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.982424 20535 ts_manager.cc:194] Registered new tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:20:49.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.987002 21124 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:20:49.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.987190 21124 heartbeater.cc:461] Registering TS with master...
18:20:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.987900 21124 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:20:49.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.989066 20700 ts_manager.cc:194] Registered new tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:20:49.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:49.990229 20700 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.66:33041
18:20:50.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.245901 20964 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:20:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:50.587733 21133 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:20:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:50.588467 21133 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:20:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:50.588737 21133 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:20:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:50.604058 21133 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:20:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:50.604432 21133 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.67
18:20:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.621222 21133 tablet_server_runner.cc:78] Tablet server non-default flags:
18:20:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/logs
18:20:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:50.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.67:0
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.67
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:20:50.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:20:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:20:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:20:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.67
18:20:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:20:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:20:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:20:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:20:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:20:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:20:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:20:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:20:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.624101 21133 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:20:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.627205 21133 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:20:50.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:50.644604 21140 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:50.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:50.644920 21139 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:50.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:50.646428 21143 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.648413 21133 server_base.cc:1061] running on GCE node
18:20:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.649374 21133 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:20:50.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.650704 21133 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:20:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.652030 21133 hybrid_clock.cc:648] HybridClock initialized: now 1786472450651887 us; error 65 us; skew 500 ppm
18:20:50.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.652459 21133 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:20:50.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.656391 21133 webserver.cc:533] Webserver started at http://127.20.1.67:46629/ using document root <none> and password file <none>
18:20:50.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.657542 21133 fs_manager.cc:362] Metadata directory not provided
18:20:50.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.657698 21133 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:20:50.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.658247 21133 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
18:20:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.661883 21133 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/instance:
18:20:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5a72ae51147432e815137a6da2c75ff"
18:20:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:50 on dist-test-slave-6k22"
18:20:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.662994 21133 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal/instance:
18:20:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5a72ae51147432e815137a6da2c75ff"
18:20:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:50 on dist-test-slave-6k22"
18:20:50.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.670221 21133 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.000s
18:20:50.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.675619 21150 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:20:50.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.677345 21133 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.677629 21133 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5a72ae51147432e815137a6da2c75ff"
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:50 on dist-test-slave-6k22"
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.678084 21133 fs_report.cc:389] FS layout report
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/data
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:20:50.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:20:50.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:20:50.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:20:50.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:20:50.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:20:50.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:20:50.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:20:50.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.723510 21133 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:20:50.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.725358 21133 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:20:50.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.725965 21133 kserver.cc:163] Server-wide thread pool size limit: 3276
18:20:50.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.727792 21133 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:20:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.731134 21133 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:20:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.731309 21133 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
18:20:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.731452 21133 ts_tablet_manager.cc:616] Registered 0 tablets
18:20:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.731694 21133 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
18:20:50.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.837922 21133 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.67:41125
18:20:50.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.837965 21278 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.67:41125 every 8 connection(s)
18:20:50.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.839948 21133 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:20:50.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.848817 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 21133
18:20:50.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.849128 20485 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal/instance
18:20:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.877593 21279 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:20:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.878086 21279 heartbeater.cc:461] Registering TS with master...
18:20:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.879889 21279 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:20:50.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.883509 21281 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:20:50.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.883713 21281 heartbeater.cc:461] Registering TS with master...
18:20:50.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.883507 20700 ts_manager.cc:194] Registered new tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:20:50.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.884646 21281 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:20:50.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.885236 20700 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.67:53765
18:20:50.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.885936 21282 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:20:50.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.886114 21282 heartbeater.cc:461] Registering TS with master...
18:20:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.887023 21282 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:20:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.887068 20535 ts_manager.cc:194] Registered new tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:20:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.890537 20613 ts_manager.cc:194] Registered new tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:20:50.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.902969 20485 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
18:20:50.905 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
18:20:50.906 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

18:20:50.918 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.20.1.126"
      port: 36795
    }
    bound_http_address {
      host: "127.20.1.126"
      port: 33441
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.20.1.125"
      port: 35653
    }
    bound_http_address {
      host: "127.20.1.125"
      port: 38031
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.20.1.124"
      port: 35869
    }
    bound_http_address {
      host: "127.20.1.124"
      port: 40603
    }
  }
}

18:20:50.922 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

18:20:50.925 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.20.1.65"
      port: 36397
    }
    bound_http_address {
      host: "127.20.1.65"
      port: 46555
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.20.1.66"
      port: 40881
    }
    bound_http_address {
      host: "127.20.1.66"
      port: 40147
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.20.1.67"
      port: 41125
    }
    bound_http_address {
      host: "127.20.1.67"
      port: 46629
    }
  }
}

18:20:50.925 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
18:20:50.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:50.993331 21124 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:20:51.156 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttempt(RetryRule.java:225)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:246)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
18:20:51.219 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
18:20:51.389 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.1.126:36795(127.20.1.126:36795)] connecting to peer
18:20:51.443 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.1.124:35869(127.20.1.124:35869)] connecting to peer
18:20:51.445 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.1.125:35653(127.20.1.125:35653)] connecting to peer
18:20:51.466 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.1.124:35869(127.20.1.124:35869)] Successfully connected to peer
18:20:51.468 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.20.1.125:35653(127.20.1.125:35653)] Successfully connected to peer
18:20:51.468 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.1.126:36795(127.20.1.126:36795)] Successfully connected to peer
18:20:51.607 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.20.1.124
18:20:51.607 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.20.1.126
18:20:51.608 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism PLAIN chosen for peer 127.20.1.125
18:20:51.847 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x5b5b634b, L:/127.0.0.1:43392 - R:/127.20.1.126:36795] using SASL/PLAIN
18:20:51.847 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xd1e961d1, L:/127.0.0.1:42784 - R:/127.20.1.125:35653] using SASL/PLAIN
18:20:51.847 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xb7671f34, L:/127.0.0.1:55364 - R:/127.20.1.124:35869] using SASL/PLAIN
18:20:51.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:51.889683 21279 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:20:51.893 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
18:20:51.894 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=549), Traces: [0ms] refreshing cache from master, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.1.126:36795, [93ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.1.124:35869, [95ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.1.125:35653, [510ms] Sub RPC ConnectToMaster: received response from server master-127.20.1.126:36795: OK, [510ms] Sub RPC ConnectToMaster: received response from server master-127.20.1.125:35653: OK, [511ms] Sub RPC ConnectToMaster: received response from server master-127.20.1.124:35869: OK, deferred=null) after lookup
18:20:51.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:51.957185 20700 catalog_manager.cc:2283] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55364:
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330"
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:20:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:20:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
18:20:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
18:20:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:20:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
18:20:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
18:20:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
18:20:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
18:20:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:20:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:20:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:20:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
18:20:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
18:20:51.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:51.959476 20700 catalog_manager.cc:7099] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
18:20:52.017 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table e707167631db4dc7a1777ca94dd1a5cd
18:20:52.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.062419 21207 tablet_service.cc:1511] Processing CreateTablet for tablet c309490fd3a441c9b1a125d394f3629e (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330 [id=e707167631db4dc7a1777ca94dd1a5cd]), partition=RANGE (key) PARTITION UNBOUNDED
18:20:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.064594 21046 tablet_service.cc:1511] Processing CreateTablet for tablet c309490fd3a441c9b1a125d394f3629e (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330 [id=e707167631db4dc7a1777ca94dd1a5cd]), partition=RANGE (key) PARTITION UNBOUNDED
18:20:52.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.066004 21207 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c309490fd3a441c9b1a125d394f3629e. 1 dirs total, 0 dirs full, 0 dirs failed
18:20:52.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.066864 21046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c309490fd3a441c9b1a125d394f3629e. 1 dirs total, 0 dirs full, 0 dirs failed
18:20:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.070492 20888 tablet_service.cc:1511] Processing CreateTablet for tablet c309490fd3a441c9b1a125d394f3629e (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330 [id=e707167631db4dc7a1777ca94dd1a5cd]), partition=RANGE (key) PARTITION UNBOUNDED
18:20:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.072430 20888 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c309490fd3a441c9b1a125d394f3629e. 1 dirs total, 0 dirs full, 0 dirs failed
18:20:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.099627 21324 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Bootstrap starting.
18:20:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.101416 21323 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Bootstrap starting.
18:20:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.104081 21325 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap starting.
18:20:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.109875 21323 tablet_bootstrap.cc:654] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Neither blocks nor log segments found. Creating new log.
18:20:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.109910 21324 tablet_bootstrap.cc:654] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Neither blocks nor log segments found. Creating new log.
18:20:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.111002 21325 tablet_bootstrap.cc:654] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Neither blocks nor log segments found. Creating new log.
18:20:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.112495 21323 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Log is configured to *not* fsync() on all Append() calls
18:20:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.112468 21324 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Log is configured to *not* fsync() on all Append() calls
18:20:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.113699 21325 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Log is configured to *not* fsync() on all Append() calls
18:20:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.122632 21323 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: No bootstrap required, opened a new log
18:20:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.122781 21324 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: No bootstrap required, opened a new log
18:20:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.123025 21323 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Time spent bootstrapping tablet: real 0.026s	user 0.011s	sys 0.008s
18:20:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.123236 21324 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Time spent bootstrapping tablet: real 0.024s	user 0.007s	sys 0.011s
18:20:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.124354 21325 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: No bootstrap required, opened a new log
18:20:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.124758 21325 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Time spent bootstrapping tablet: real 0.021s	user 0.005s	sys 0.012s
18:20:52.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.133069 21323 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:52.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.133669 21323 raft_consensus.cc:385] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:20:52.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.133842 21323 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 90f4362b094e4d1681153e2008083f4e, State: Initialized, Role: FOLLOWER
18:20:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.134959 21323 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.137810 21324 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.138702 21324 raft_consensus.cc:385] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:20:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.138759 21325 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.139215 21324 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: afe5bf8ec6e44b15bad84f94f3ee7158, State: Initialized, Role: FOLLOWER
18:20:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.139621 21325 raft_consensus.cc:385] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:20:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.140031 21325 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b5a72ae51147432e815137a6da2c75ff, State: Initialized, Role: FOLLOWER
18:20:52.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.140543 21324 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.141404 21325 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.144573 21323 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Time spent starting tablet: real 0.021s	user 0.005s	sys 0.018s
18:20:52.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.148247 21324 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Time spent starting tablet: real 0.025s	user 0.001s	sys 0.020s
18:20:52.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.148741 21325 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Time spent starting tablet: real 0.024s	user 0.008s	sys 0.011s
18:20:52.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:52.196290 21127 tablet.cc:2406] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:20:52.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:52.209672 20968 tablet.cc:2406] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:20:52.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:52.355140 21283 tablet.cc:2406] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:20:52.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.485105 21333 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:20:52.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.485468 21333 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:52.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.488292 21333 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.501183 21069 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "b5a72ae51147432e815137a6da2c75ff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" is_pre_election: true
18:20:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.501998 21069 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b5a72ae51147432e815137a6da2c75ff in term 0.
18:20:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.501538 20911 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "b5a72ae51147432e815137a6da2c75ff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e" is_pre_election: true
18:20:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.502606 20911 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b5a72ae51147432e815137a6da2c75ff in term 0.
18:20:52.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.502897 21151 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: afe5bf8ec6e44b15bad84f94f3ee7158, b5a72ae51147432e815137a6da2c75ff; no voters: 
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.503654 21333 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 0 FOLLOWER]: Leader pre-election won for term 1
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.503851 21333 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.504009 21333 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 0 FOLLOWER]: Advancing to term 1
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.508132 21333 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.509766 21069 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "b5a72ae51147432e815137a6da2c75ff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158"
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.510002 21069 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 0 FOLLOWER]: Advancing to term 1
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.510231 21333 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 1 election: Requested vote from peers afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.511215 20911 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "b5a72ae51147432e815137a6da2c75ff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e"
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.511458 20911 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 0 FOLLOWER]: Advancing to term 1
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.514341 21069 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b5a72ae51147432e815137a6da2c75ff in term 1.
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.515005 21151 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: afe5bf8ec6e44b15bad84f94f3ee7158, b5a72ae51147432e815137a6da2c75ff; no voters: 
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.516058 20911 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b5a72ae51147432e815137a6da2c75ff in term 1.
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.516582 21333 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 1 FOLLOWER]: Leader election won for term 1
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.526309 21333 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 1 LEADER]: Becoming Leader. State: Replica: b5a72ae51147432e815137a6da2c75ff, State: Running, Role: LEADER
18:20:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.527061 21333 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:52.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.534591 20699 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff reported cstate change: term changed from 0 to 1, leader changed from <none> to b5a72ae51147432e815137a6da2c75ff (127.20.1.67). New cstate: current_term: 1 leader_uuid: "b5a72ae51147432e815137a6da2c75ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: UNKNOWN } } }
18:20:52.695 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table e707167631db4dc7a1777ca94dd1a5cd
18:20:52.716 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:52.722 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299994, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:20:52.724 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [9ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=null) after lookup
18:20:52.744 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:20:52.747 [DEBUG - kudu-nio-3] (Connection.java:757) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] Successfully connected to peer
18:20:52.850 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0xc365407c, L:/127.0.0.1:56428 - R:/127.20.1.67:41125] using TOKEN/null
18:20:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.865346 20911 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 1 FOLLOWER]: Refusing update from remote peer b5a72ae51147432e815137a6da2c75ff: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:20:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.865854 21069 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 1 FOLLOWER]: Refusing update from remote peer b5a72ae51147432e815137a6da2c75ff: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:20:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.866837 21333 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [LEADER]: Connected to new peer: Peer: permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:20:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:52.867313 21336 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [LEADER]: Connected to new peer: Peer: permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
18:20:53.020 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:20:53.022 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.1.67:41125
18:20:53.024 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:20:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:53.024659 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 21133
18:20:53.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:53.045375 21274 generic_service.cc:196] Checking for leaks (request via RPC)
18:20:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:53.149185 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 21133
18:20:53.177 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:20:53.181 [DEBUG - kudu-nio-3] (Connection.java:239) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] handling channelInactive
18:20:53.182 [DEBUG - kudu-nio-3] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state READY due to: connection closed
18:20:53.182 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: connection closed
18:20:53.190 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:53.190 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:20:53.191 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=13), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [5ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [13ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@771508530(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@3b1e8503 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10b533f6 -> wakeup thread Time-limited test)) after lookup
18:20:53.191 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:20:53.194 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.194 [DEBUG - kudu-nio-4] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.194 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.210 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:53.210 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:20:53.210 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=33), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [5ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [13ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [14ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [17ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@771508530(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@3b1e8503 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1120466114 after retry RPC@2110614113), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10b533f6 -> wakeup thread Time-limited test -> (continuation of Deferred@1120466114 after retry RPC@2110614113))) after lookup
18:20:53.211 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:20:53.213 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.213 [DEBUG - kudu-nio-5] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.213 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.230 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:53.231 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:20:53.231 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=53), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [5ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [13ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [14ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [17ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [34ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [36ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [36ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [53ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@771508530(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@3b1e8503 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1120466114 after retry RPC@2110614113) -> (continuation of Deferred@1824351457 after retry RPC@569672521), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10b533f6 -> wakeup thread Time-limited test -> (continuation of Deferred@1120466114 after retry RPC@2110614113) -> (continuation of Deferred@1824351457 after retry RPC@569672521))) after lookup
18:20:53.232 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:20:53.234 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.234 [DEBUG - kudu-nio-6] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.234 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.250 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:53.250 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:20:53.251 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=73), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [5ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [13ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [14ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [17ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [34ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [36ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [36ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [53ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [55ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [57ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [57ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [72ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@771508530(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@3b1e8503 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1120466114 after retry RPC@2110614113) -> (continuation of Deferred@1824351457 after retry RPC@569672521) -> (continuation of Deferred@2057206141 after retry RPC@1689286346), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10b533f6 -> wakeup thread Time-limited test -> (continuation of Deferred@1120466114 after retry RPC@2110614113) -> (continuation of Deferred@1824351457 after retry RPC@569672521) -> (continuation of Deferred@2057206141 after retry RPC@1689286346))) after lookup
18:20:53.251 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:20:53.253 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.253 [DEBUG - kudu-nio-7] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.253 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.430 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:53.431 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:20:53.432 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=253), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [5ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [13ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [14ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [17ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [34ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [36ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [36ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [53ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [55ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [57ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [57ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [72ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [74ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [76ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [77ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [250ms] refreshing cache from master, [250ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [253ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@771508530(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@3b1e8503 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1120466114 after retry RPC@2110614113) -> (continuation of Deferred@1824351457 after retry RPC@569672521) -> (continuation of Deferred@2057206141 after retry RPC@1689286346) -> (continuation of Deferred@1656011944 after retry RPC@812657309), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10b533f6 -> wakeup thread Time-limited test -> (continuation of Deferred@1120466114 after retry RPC@2110614113) -> (continuation of Deferred@1824351457 after retry RPC@569672521) -> (continuation of Deferred@2057206141 after retry RPC@1689286346) -> (continuation of Deferred@1656011944 after retry RPC@812657309))) after lookup
18:20:53.432 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:20:53.434 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.434 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:53.434 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:20:54.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.458096 21364 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 1 FOLLOWER]: Starting pre-election (detected failure of leader b5a72ae51147432e815137a6da2c75ff)
18:20:54.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.458390 21364 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:54.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.459409 21366 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader b5a72ae51147432e815137a6da2c75ff)
18:20:54.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.459656 21366 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:54.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.460368 21364 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:20:54.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.461424 21366 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:54.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:54.463593 20834 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:20:54.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:54.466419 20834 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:20:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:54.467550 20994 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:20:54.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:54.469954 20994 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:20:54.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.471135 21069 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "90f4362b094e4d1681153e2008083f4e" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" is_pre_election: true
18:20:54.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.471403 21069 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 90f4362b094e4d1681153e2008083f4e in term 1.
18:20:54.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.472107 20830 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, afe5bf8ec6e44b15bad84f94f3ee7158; no voters: b5a72ae51147432e815137a6da2c75ff
18:20:54.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.472565 21364 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 1 FOLLOWER]: Leader pre-election won for term 2
18:20:54.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.472704 21364 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 1 FOLLOWER]: Starting leader election (detected failure of leader b5a72ae51147432e815137a6da2c75ff)
18:20:54.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.472836 21364 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 1 FOLLOWER]: Advancing to term 2
18:20:54.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.475003 20911 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e" is_pre_election: true
18:20:54.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.476068 21364 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:54.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.476475 20911 raft_consensus.cc:2393] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate afe5bf8ec6e44b15bad84f94f3ee7158 in current term 2: Already voted for candidate 90f4362b094e4d1681153e2008083f4e in this term.
18:20:54.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.476922 21364 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 2 election: Requested vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:20:54.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.477531 21069 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "90f4362b094e4d1681153e2008083f4e" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158"
18:20:54.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.477746 21069 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 1 FOLLOWER]: Advancing to term 2
18:20:54.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.477720 20994 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: afe5bf8ec6e44b15bad84f94f3ee7158; no voters: 90f4362b094e4d1681153e2008083f4e, b5a72ae51147432e815137a6da2c75ff
18:20:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:54.478678 20834 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:20:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.480086 21069 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 90f4362b094e4d1681153e2008083f4e in term 2.
18:20:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.480279 21366 raft_consensus.cc:2749] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
18:20:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.480597 20830 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, afe5bf8ec6e44b15bad84f94f3ee7158; no voters: b5a72ae51147432e815137a6da2c75ff
18:20:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.480962 21364 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 2 FOLLOWER]: Leader election won for term 2
18:20:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.481518 21364 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 2 LEADER]: Becoming Leader. State: Replica: 90f4362b094e4d1681153e2008083f4e, State: Running, Role: LEADER
18:20:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.482080 21364 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:54.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:54.486316 20698 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e reported cstate change: term changed from 1 to 2, leader changed from b5a72ae51147432e815137a6da2c75ff (127.20.1.67) to 90f4362b094e4d1681153e2008083f4e (127.20.1.65). New cstate: current_term: 2 leader_uuid: "90f4362b094e4d1681153e2008083f4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: HEALTHY } } }
18:20:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:54.930228 20834 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e -> Peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Couldn't send request to peer b5a72ae51147432e815137a6da2c75ff. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:20:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.055824 21069 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 2 FOLLOWER]: Refusing update from remote peer 90f4362b094e4d1681153e2008083f4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
18:20:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.056633 21364 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
18:20:55.171 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:55.172 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:55.173 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1995), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [5ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: connection closed, [5ms] delaying RPC due to: Network error: connection closed, [10ms] refreshing cache from master, [10ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [13ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [14ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [17ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [29ms] refreshing cache from master, [29ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [32ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [34ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [36ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [36ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [50ms] refreshing cache from master, [50ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [53ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [55ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [57ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [57ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [69ms] refreshing cache from master, [70ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [72ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [74ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [76ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [77ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [250ms] refreshing cache from master, [250ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [253ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [255ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [257ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [257ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [1990ms] refreshing cache from master, [1990ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1994ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@771508530(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@3b1e8503 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1120466114 after retry RPC@2110614113) -> (continuation of Deferred@1824351457 after retry RPC@569672521) -> (continuation of Deferred@2057206141 after retry RPC@1689286346) -> (continuation of Deferred@1656011944 after retry RPC@812657309) -> (continuation of Deferred@410930868 after retry RPC@703314916), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@10b533f6 -> wakeup thread Time-limited test -> (continuation of Deferred@1120466114 after retry RPC@2110614113) -> (continuation of Deferred@1824351457 after retry RPC@569672521) -> (continuation of Deferred@2057206141 after retry RPC@1689286346) -> (continuation of Deferred@1656011944 after retry RPC@812657309) -> (continuation of Deferred@410930868 after retry RPC@703314916))) after lookup
18:20:55.174 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:20:55.175 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] Successfully connected to peer
18:20:55.203 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x74695558, L:/127.0.0.1:40674 - R:/127.20.1.65:36397] using TOKEN/null
18:20:55.231 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.1.67:41125
18:20:55.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.233835 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/logs
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.67:41125
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.67
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46629
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.67
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:20:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:55.592520 21402 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:20:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:55.592844 21402 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:20:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:55.592919 21402 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:20:55.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:55.601833 21402 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:20:55.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:55.602007 21402 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.67
18:20:55.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.613368 21402 tablet_server_runner.cc:78] Tablet server non-default flags:
18:20:55.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/logs
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.67:41125
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.67
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46629
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:20:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.67
18:20:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:20:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:20:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:20:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:20:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:20:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:20:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:20:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:20:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.615805 21402 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:20:55.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.618227 21402 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:20:55.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:55.630266 21411 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:55.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:55.630632 21414 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:55.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:55.630870 21412 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.631963 21402 server_base.cc:1061] running on GCE node
18:20:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.632591 21402 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:20:55.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.633721 21402 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:20:55.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.634956 21402 hybrid_clock.cc:648] HybridClock initialized: now 1786472455634897 us; error 42 us; skew 500 ppm
18:20:55.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.635262 21402 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:20:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.637949 21402 webserver.cc:533] Webserver started at http://127.20.1.67:46629/ using document root <none> and password file <none>
18:20:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.638715 21402 fs_manager.cc:362] Metadata directory not provided
18:20:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.638829 21402 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:20:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.645045 21402 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.001s
18:20:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.648306 21422 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:20:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.649858 21402 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
18:20:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.650015 21402 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:20:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5a72ae51147432e815137a6da2c75ff"
18:20:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:50 on dist-test-slave-6k22"
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.650811 21402 fs_report.cc:389] FS layout report
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/data
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:20:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:20:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.677912 21402 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:20:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.678845 21402 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:20:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.679108 21402 kserver.cc:163] Server-wide thread pool size limit: 3276
18:20:55.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.680316 21402 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:20:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.682717 21429 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:20:55.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.688902 21402 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:20:55.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.688998 21402 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
18:20:55.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.689105 21402 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:20:55.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.692281 21402 ts_tablet_manager.cc:616] Registered 1 tablets
18:20:55.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.692370 21402 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
18:20:55.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.692898 21429 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap starting.
18:20:55.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.717432 21429 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Log is configured to *not* fsync() on all Append() calls
18:20:55.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.739156 21429 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:20:55.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.740195 21429 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap complete.
18:20:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.741688 21429 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Time spent bootstrapping tablet: real 0.049s	user 0.030s	sys 0.016s
18:20:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.748124 21429 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.749373 21429 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b5a72ae51147432e815137a6da2c75ff, State: Initialized, Role: FOLLOWER
18:20:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.750317 21429 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:55.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.752250 21429 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Time spent starting tablet: real 0.010s	user 0.011s	sys 0.001s
18:20:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.764186 21402 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.67:41125
18:20:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.764271 21550 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.67:41125 every 8 connection(s)
18:20:55.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.765699 21402 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:20:55.768 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:20:55.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.767354 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 21402
18:20:55.789 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:20:55.790 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.1.65:36397
18:20:55.791 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:20:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.791416 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 20811
18:20:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.799316 21553 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:20:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.799928 21553 heartbeater.cc:461] Registering TS with master...
18:20:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.801925 21553 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:20:55.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.805481 21551 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:20:55.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.805751 21551 heartbeater.cc:461] Registering TS with master...
18:20:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.806973 21551 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:20:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.808476 20533 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:20:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.809147 20698 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:20:55.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.811857 21554 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:20:55.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.811856 20698 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.67:33027
18:20:55.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.812309 21554 heartbeater.cc:461] Registering TS with master...
18:20:55.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.813437 21554 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:20:55.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.815883 20613 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:20:55.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.820097 20959 generic_service.cc:196] Checking for leaks (request via RPC)
18:20:55.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:55.926652 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 20811
18:20:55.972 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] handling channelInactive
18:20:55.973 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state READY due to: connection closed
18:20:55.973 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:20:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:55.973263 21567 negotiation.cc:336] Failed RPC negotiation. Trace:
18:20:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 18:20:55.930114 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.20.1.65:50443 (local address 127.20.1.67:41125)
18:20:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 18:20:55.932286 (+  2172us) server_negotiation.cc:207] Beginning negotiation
18:20:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 18:20:55.932298 (+    12us) server_negotiation.cc:400] Waiting for connection header
18:20:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 18:20:55.932340 (+    42us) server_negotiation.cc:408] Connection header received
18:20:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 18:20:55.932477 (+   137us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
18:20:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 18:20:55.932489 (+    12us) server_negotiation.cc:462] Received NEGOTIATE request from client
18:20:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 18:20:55.932827 (+   338us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
18:20:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 18:20:55.933222 (+   395us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
18:20:55.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 18:20:55.971553 (+ 38331us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.20.1.65:50443: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
18:20:55.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1943,"thread_start_us":1786,"threads_started":1}
18:20:55.974 [DEBUG - Time-limited test] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:20:55.975 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:55.975 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:55.976 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:55.990 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:55.991 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:55.991 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1502584797(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e73c62c -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@287123f0 -> wakeup thread Time-limited test)) after lookup
18:20:55.992 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:20:55.993 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:55.993 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:55.993 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.010 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:56.011 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:56.012 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [17ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [19ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1502584797(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e73c62c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1838872086 after retry RPC@1289037558), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@287123f0 -> wakeup thread Time-limited test -> (continuation of Deferred@1838872086 after retry RPC@1289037558))) after lookup
18:20:56.013 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:20:56.014 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.014 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.014 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.050 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:56.051 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:56.052 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=77), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [17ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [19ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [38ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [40ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [40ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [76ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1502584797(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e73c62c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1838872086 after retry RPC@1289037558) -> (continuation of Deferred@1941551275 after retry RPC@373215932), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@287123f0 -> wakeup thread Time-limited test -> (continuation of Deferred@1838872086 after retry RPC@1289037558) -> (continuation of Deferred@1941551275 after retry RPC@373215932))) after lookup
18:20:56.053 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:20:56.054 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.054 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.054 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.090 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:56.090 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:56.091 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=117), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [17ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [19ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [38ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [40ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [40ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [76ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [78ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [80ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [80ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [116ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1502584797(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e73c62c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1838872086 after retry RPC@1289037558) -> (continuation of Deferred@1941551275 after retry RPC@373215932) -> (continuation of Deferred@929953964 after retry RPC@1470523343), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@287123f0 -> wakeup thread Time-limited test -> (continuation of Deferred@1838872086 after retry RPC@1289037558) -> (continuation of Deferred@1941551275 after retry RPC@373215932) -> (continuation of Deferred@929953964 after retry RPC@1470523343))) after lookup
18:20:56.092 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:20:56.093 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.093 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.093 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.431 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:56.431 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:56.432 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=458), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [17ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [19ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [38ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [40ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [40ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [76ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [78ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [80ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [80ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [116ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [117ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [119ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [119ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [453ms] refreshing cache from master, [453ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [456ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1502584797(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e73c62c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1838872086 after retry RPC@1289037558) -> (continuation of Deferred@1941551275 after retry RPC@373215932) -> (continuation of Deferred@929953964 after retry RPC@1470523343) -> (continuation of Deferred@198169956 after retry RPC@1865630669), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@287123f0 -> wakeup thread Time-limited test -> (continuation of Deferred@1838872086 after retry RPC@1289037558) -> (continuation of Deferred@1941551275 after retry RPC@373215932) -> (continuation of Deferred@929953964 after retry RPC@1470523343) -> (continuation of Deferred@198169956 after retry RPC@1865630669))) after lookup
18:20:56.432 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:20:56.434 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.434 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.434 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:20:56.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:56.817831 21551 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:20:57.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.137877 21568 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 90f4362b094e4d1681153e2008083f4e)
18:20:57.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.138094 21568 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:57.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.138877 21568 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:57.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:57.144819 20994 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397): Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111)
18:20:57.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.148047 21502 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "b5a72ae51147432e815137a6da2c75ff" is_pre_election: true
18:20:57.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.148583 21502 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate afe5bf8ec6e44b15bad84f94f3ee7158 in term 1.
18:20:57.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.149595 20994 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: afe5bf8ec6e44b15bad84f94f3ee7158, b5a72ae51147432e815137a6da2c75ff; no voters: 90f4362b094e4d1681153e2008083f4e
18:20:57.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.149983 21568 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 2 FOLLOWER]: Leader pre-election won for term 3
18:20:57.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.150111 21568 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 90f4362b094e4d1681153e2008083f4e)
18:20:57.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.150198 21568 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 2 FOLLOWER]: Advancing to term 3
18:20:57.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.153126 21568 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:57.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.153718 21568 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 3 election: Requested vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:57.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.154345 21502 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "b5a72ae51147432e815137a6da2c75ff"
18:20:57.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.154572 21502 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 1 FOLLOWER]: Advancing to term 3
18:20:57.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:57.155442 20994 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397): Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111)
18:20:57.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.158496 21502 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate afe5bf8ec6e44b15bad84f94f3ee7158 in term 3.
18:20:57.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.159068 20994 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: afe5bf8ec6e44b15bad84f94f3ee7158, b5a72ae51147432e815137a6da2c75ff; no voters: 90f4362b094e4d1681153e2008083f4e
18:20:57.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.159389 21568 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 3 FOLLOWER]: Leader election won for term 3
18:20:57.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.159911 21568 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 3 LEADER]: Becoming Leader. State: Replica: afe5bf8ec6e44b15bad84f94f3ee7158, State: Running, Role: LEADER
18:20:57.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.160743 21568 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.164495 20698 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 reported cstate change: term changed from 2 to 3, leader changed from 90f4362b094e4d1681153e2008083f4e (127.20.1.65) to afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66). New cstate: current_term: 3 leader_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: UNKNOWN } } }
18:20:57.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.744642 21502 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 3 FOLLOWER]: Refusing update from remote peer afe5bf8ec6e44b15bad84f94f3ee7158: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 9. (index mismatch)
18:20:57.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:57.744640 20994 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 -> Peer 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397): Couldn't send request to peer 90f4362b094e4d1681153e2008083f4e. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:20:57.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:57.745697 21588 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
18:20:58.172 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:58.172 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:58.173 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=2199), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [16ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [17ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [19ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [33ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [36ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [38ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [40ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [40ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [76ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [78ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [80ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [80ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [113ms] refreshing cache from master, [113ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [116ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [117ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [119ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [119ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [453ms] refreshing cache from master, [453ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [456ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [458ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [460ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [460ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2193ms] refreshing cache from master, [2193ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [2197ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1502584797(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e73c62c -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1838872086 after retry RPC@1289037558) -> (continuation of Deferred@1941551275 after retry RPC@373215932) -> (continuation of Deferred@929953964 after retry RPC@1470523343) -> (continuation of Deferred@198169956 after retry RPC@1865630669) -> (continuation of Deferred@368193697 after retry RPC@2128066726), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@287123f0 -> wakeup thread Time-limited test -> (continuation of Deferred@1838872086 after retry RPC@1289037558) -> (continuation of Deferred@1941551275 after retry RPC@373215932) -> (continuation of Deferred@929953964 after retry RPC@1470523343) -> (continuation of Deferred@198169956 after retry RPC@1865630669) -> (continuation of Deferred@368193697 after retry RPC@2128066726))) after lookup
18:20:58.175 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:20:58.177 [DEBUG - kudu-nio-0] (Connection.java:757) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] Successfully connected to peer
18:20:58.215 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x96aae950, L:/127.0.0.1:42146 - R:/127.20.1.66:40881] using TOKEN/null
18:20:58.247 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.1.65:36397
18:20:58.248 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.249188 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/logs
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:58.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.65:36397
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.65
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46555
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.65
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:58.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:20:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:58.604861 21603 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:20:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:58.605243 21603 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:20:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:58.605360 21603 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:20:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:58.614303 21603 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:20:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:58.614523 21603 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.65
18:20:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.625090 21603 tablet_server_runner.cc:78] Tablet server non-default flags:
18:20:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/logs
18:20:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:20:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:20:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:20:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:20:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:20:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:20:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data
18:20:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.65:36397
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.65
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46555
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.65
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:20:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:20:58.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.627324 21603 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:20:58.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.629346 21603 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:20:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:58.640290 21612 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:58.640290 21611 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:20:58.640381 21614 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:20:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.641458 21603 server_base.cc:1061] running on GCE node
18:20:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.642104 21603 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:20:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.643103 21603 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:20:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.644335 21603 hybrid_clock.cc:648] HybridClock initialized: now 1786472458644266 us; error 55 us; skew 500 ppm
18:20:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.644713 21603 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:20:58.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.647063 21603 webserver.cc:533] Webserver started at http://127.20.1.65:46555/ using document root <none> and password file <none>
18:20:58.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.647929 21603 fs_manager.cc:362] Metadata directory not provided
18:20:58.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.648092 21603 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:20:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.653797 21603 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
18:20:58.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.657015 21620 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:20:58.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.658578 21603 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
18:20:58.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.658773 21603 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:20:58.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90f4362b094e4d1681153e2008083f4e"
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.659839 21603 fs_report.cc:389] FS layout report
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/data
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:20:58.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:20:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.707851 21603 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:20:58.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.708820 21603 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:20:58.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.709108 21603 kserver.cc:163] Server-wide thread pool size limit: 3276
18:20:58.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.710302 21603 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:20:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.712700 21629 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:20:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.718907 21603 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:20:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.719039 21603 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
18:20:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.719189 21603 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:20:58.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.722369 21603 ts_tablet_manager.cc:616] Registered 1 tablets
18:20:58.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.722503 21603 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.001s
18:20:58.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.722818 21629 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Bootstrap starting.
18:20:58.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.749938 21629 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Log is configured to *not* fsync() on all Append() calls
18:20:58.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.782056 21629 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:20:58.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.782891 21629 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Bootstrap complete.
18:20:58.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.784804 21629 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Time spent bootstrapping tablet: real 0.062s	user 0.039s	sys 0.012s
18:20:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.792824 21629 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.793730 21629 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 90f4362b094e4d1681153e2008083f4e, State: Initialized, Role: FOLLOWER
18:20:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.794541 21629 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:20:58.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.796723 21629 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Time spent starting tablet: real 0.012s	user 0.005s	sys 0.005s
18:20:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.801080 21603 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.65:36397
18:20:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.801148 21754 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.65:36397 every 8 connection(s)
18:20:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.802500 21603 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:20:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.812471 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 21603
18:20:58.813 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:20:58.828 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:20:58.829 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.1.66:40881
18:20:58.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:20:58.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.830008 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 20975
18:20:58.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.856472 21756 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:20:58.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.857302 21756 heartbeater.cc:461] Registering TS with master...
18:20:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.860219 21756 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:20:58.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.865875 20534 ts_manager.cc:194] Re-registered known tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.869490 21755 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:20:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.869761 21755 heartbeater.cc:461] Registering TS with master...
18:20:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.870673 21755 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:20:58.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.872063 21757 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:20:58.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.872251 21757 heartbeater.cc:461] Registering TS with master...
18:20:58.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.872809 20699 ts_manager.cc:194] Re-registered known tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:58.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.873106 21757 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:20:58.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.874717 21119 generic_service.cc:196] Checking for leaks (request via RPC)
18:20:58.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.875097 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.65:40163
18:20:58.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.877276 20613 ts_manager.cc:194] Re-registered known tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:20:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:58.984427 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 20975
18:20:59.015 [DEBUG - kudu-nio-0] (Connection.java:239) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] handling channelInactive
18:20:59.015 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:20:59.015 [DEBUG - kudu-nio-0] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state READY due to: connection closed
18:20:59.016 [DEBUG - Time-limited test] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:20:59.017 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.017 [DEBUG - kudu-nio-1] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.018 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.031 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:59.031 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:59.032 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1688067322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5e6a53d8 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7a47f934 -> wakeup thread Time-limited test)) after lookup
18:20:59.033 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:20:59.034 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.034 [DEBUG - kudu-nio-2] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.035 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.051 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:59.051 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:59.052 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=36), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [18ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1688067322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5e6a53d8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@400749322 after retry RPC@1386817179), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7a47f934 -> wakeup thread Time-limited test -> (continuation of Deferred@400749322 after retry RPC@1386817179))) after lookup
18:20:59.052 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:20:59.053 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.053 [DEBUG - kudu-nio-3] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.053 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.071 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:59.072 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:59.072 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=56), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [18ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [36ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [37ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [37ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [55ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1688067322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5e6a53d8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@400749322 after retry RPC@1386817179) -> (continuation of Deferred@1535907565 after retry RPC@1082814201), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7a47f934 -> wakeup thread Time-limited test -> (continuation of Deferred@400749322 after retry RPC@1386817179) -> (continuation of Deferred@1535907565 after retry RPC@1082814201))) after lookup
18:20:59.073 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:20:59.074 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.075 [DEBUG - kudu-nio-4] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.075 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.131 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:59.131 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:59.132 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=116), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [18ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [36ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [37ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [37ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [55ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [57ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [59ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [59ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [111ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [115ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1688067322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5e6a53d8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@400749322 after retry RPC@1386817179) -> (continuation of Deferred@1535907565 after retry RPC@1082814201) -> (continuation of Deferred@1375298028 after retry RPC@1108811916), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7a47f934 -> wakeup thread Time-limited test -> (continuation of Deferred@400749322 after retry RPC@1386817179) -> (continuation of Deferred@1535907565 after retry RPC@1082814201) -> (continuation of Deferred@1375298028 after retry RPC@1108811916))) after lookup
18:20:59.132 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:20:59.133 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.133 [DEBUG - kudu-nio-5] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.134 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.570 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:20:59.570 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:20:59.571 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=555), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [18ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [36ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [37ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [37ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [55ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [57ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [59ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [59ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [111ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [115ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [116ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [117ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [118ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [550ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [554ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1688067322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5e6a53d8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@400749322 after retry RPC@1386817179) -> (continuation of Deferred@1535907565 after retry RPC@1082814201) -> (continuation of Deferred@1375298028 after retry RPC@1108811916) -> (continuation of Deferred@1742773348 after retry RPC@478724948), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7a47f934 -> wakeup thread Time-limited test -> (continuation of Deferred@400749322 after retry RPC@1386817179) -> (continuation of Deferred@1535907565 after retry RPC@1082814201) -> (continuation of Deferred@1375298028 after retry RPC@1108811916) -> (continuation of Deferred@1742773348 after retry RPC@478724948))) after lookup
18:20:59.571 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:20:59.572 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.573 [DEBUG - kudu-nio-6] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.573 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:20:59.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:20:59.879837 21755 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:21:00.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.139055 21774 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 3 FOLLOWER]: Starting pre-election (detected failure of leader afe5bf8ec6e44b15bad84f94f3ee7158)
18:21:00.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.139285 21774 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.141358 21774 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.143287 21423 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:00.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.147657 21423 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881): Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:00.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.153381 21702 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "b5a72ae51147432e815137a6da2c75ff" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e" is_pre_election: true
18:21:00.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.153962 21702 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b5a72ae51147432e815137a6da2c75ff in term 2.
18:21:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.154978 21425 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, b5a72ae51147432e815137a6da2c75ff; no voters: afe5bf8ec6e44b15bad84f94f3ee7158
18:21:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.155449 21774 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 3 FOLLOWER]: Leader pre-election won for term 4
18:21:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.155625 21774 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 3 FOLLOWER]: Starting leader election (detected failure of leader afe5bf8ec6e44b15bad84f94f3ee7158)
18:21:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.155725 21774 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 3 FOLLOWER]: Advancing to term 4
18:21:00.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.157742 21774 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:00.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.158408 21774 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 4 election: Requested vote from peers afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:00.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.159058 21702 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "b5a72ae51147432e815137a6da2c75ff" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e"
18:21:00.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.159283 21702 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 2 FOLLOWER]: Advancing to term 4
18:21:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.159888 21423 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881): Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:00.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.162969 21702 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b5a72ae51147432e815137a6da2c75ff in term 4.
18:21:00.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.163401 21425 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, b5a72ae51147432e815137a6da2c75ff; no voters: afe5bf8ec6e44b15bad84f94f3ee7158
18:21:00.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.163740 21774 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 4 FOLLOWER]: Leader election won for term 4
18:21:00.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.164300 21774 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 4 LEADER]: Becoming Leader. State: Replica: b5a72ae51147432e815137a6da2c75ff, State: Running, Role: LEADER
18:21:00.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.164851 21774 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.168978 20699 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff reported cstate change: term changed from 3 to 4, leader changed from afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66) to b5a72ae51147432e815137a6da2c75ff (127.20.1.67). New cstate: current_term: 4 leader_uuid: "b5a72ae51147432e815137a6da2c75ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: UNKNOWN } } }
18:21:00.371 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:00.371 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:00.372 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=1356), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [10ms] refreshing cache from master, [11ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [18ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [31ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [36ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [37ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [37ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [51ms] refreshing cache from master, [51ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [55ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [57ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [59ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [59ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [111ms] refreshing cache from master, [111ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [115ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [116ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [117ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [118ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [550ms] refreshing cache from master, [551ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [554ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [555ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [557ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [557ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [1351ms] refreshing cache from master, [1351ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1355ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1688067322(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5e6a53d8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@400749322 after retry RPC@1386817179) -> (continuation of Deferred@1535907565 after retry RPC@1082814201) -> (continuation of Deferred@1375298028 after retry RPC@1108811916) -> (continuation of Deferred@1742773348 after retry RPC@478724948) -> (continuation of Deferred@1079390019 after retry RPC@446617214), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@7a47f934 -> wakeup thread Time-limited test -> (continuation of Deferred@400749322 after retry RPC@1386817179) -> (continuation of Deferred@1535907565 after retry RPC@1082814201) -> (continuation of Deferred@1375298028 after retry RPC@1108811916) -> (continuation of Deferred@1742773348 after retry RPC@478724948) -> (continuation of Deferred@1079390019 after retry RPC@446617214))) after lookup
18:21:00.372 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:00.373 [DEBUG - kudu-nio-7] (Connection.java:757) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] Successfully connected to peer
18:21:00.401 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0xa04b2260, L:/127.0.0.1:45690 - R:/127.20.1.67:41125] using TOKEN/null
18:21:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.408460 21702 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 4 FOLLOWER]: Refusing update from remote peer b5a72ae51147432e815137a6da2c75ff: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
18:21:00.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.409010 21423 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff -> Peer afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881): Couldn't send request to peer afe5bf8ec6e44b15bad84f94f3ee7158. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:00.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.409664 21781 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [LEADER]: Connected to new peer: Peer: permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
18:21:00.444 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.1.66:40881
18:21:00.444 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.445278 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/logs
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.66:40881
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.66
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40147
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.66
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:00.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.805063 21808 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:00.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.805454 21808 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:00.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.805575 21808 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:00.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.814659 21808 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:00.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.814949 21808 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.66
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.825646 21808 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/logs
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.66:40881
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:21:00.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.66
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40147
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.66
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:21:00.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:21:00.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.828212 21808 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:00.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.830359 21808 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:21:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.840654 21815 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.840878 21816 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:00.842435 21819 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:00.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.843344 21808 server_base.cc:1061] running on GCE node
18:21:00.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.844080 21808 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:00.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.845216 21808 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:00.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.846485 21808 hybrid_clock.cc:648] HybridClock initialized: now 1786472460846438 us; error 32 us; skew 500 ppm
18:21:00.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.846966 21808 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:21:00.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.849466 21808 webserver.cc:533] Webserver started at http://127.20.1.66:40147/ using document root <none> and password file <none>
18:21:00.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.850309 21808 fs_manager.cc:362] Metadata directory not provided
18:21:00.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.850485 21808 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:00.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.856842 21808 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
18:21:00.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.859884 21826 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:00.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.861356 21808 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.861557 21808 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afe5bf8ec6e44b15bad84f94f3ee7158"
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.862383 21808 fs_report.cc:389] FS layout report
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/data
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.878275 21808 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.879247 21808 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.879652 21808 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.880764 21808 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.882870 21833 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:21:00.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.889056 21808 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:00.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.889200 21808 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
18:21:00.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.889348 21808 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:00.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.892472 21808 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:00.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.892614 21808 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
18:21:00.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.892980 21833 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Bootstrap starting.
18:21:00.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.917863 21833 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Log is configured to *not* fsync() on all Append() calls
18:21:00.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.938691 21833 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:00.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.940209 21833 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Bootstrap complete.
18:21:00.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.941948 21833 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Time spent bootstrapping tablet: real 0.049s	user 0.035s	sys 0.012s
18:21:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.948829 21833 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.950011 21833 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: afe5bf8ec6e44b15bad84f94f3ee7158, State: Initialized, Role: FOLLOWER
18:21:00.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.950846 21833 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.952661 21833 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Time spent starting tablet: real 0.011s	user 0.011s	sys 0.001s
18:21:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.971027 21808 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.66:40881
18:21:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.971076 21965 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.66:40881 every 8 connection(s)
18:21:00.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.972949 21808 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:21:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:00.978597 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 21808
18:21:00.979 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:01.008 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:01.032 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.1.67:41125
18:21:01.033 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.032279 21967 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:21:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.032738 21967 heartbeater.cc:461] Registering TS with master...
18:21:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.033583 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 21402
18:21:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.034314 21967 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:21:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.037412 21966 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:21:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.037616 21966 heartbeater.cc:461] Registering TS with master...
18:21:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.038719 21966 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:21:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.040432 20534 ts_manager.cc:194] Re-registered known tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.041623 20699 ts_manager.cc:194] Re-registered known tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.043401 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.66:40005
18:21:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.054842 21969 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:21:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.055056 21969 heartbeater.cc:461] Registering TS with master...
18:21:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.055684 21969 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:21:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.058050 20613 ts_manager.cc:194] Re-registered known tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:01.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.062826 21546 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:01.168526 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 21402
18:21:01.194 [DEBUG - kudu-nio-7] (Connection.java:239) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] handling channelInactive
18:21:01.194 [DEBUG - kudu-nio-7] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state READY due to: connection closed
18:21:01.195 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:01.196 [DEBUG - Time-limited test] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:01.197 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.197 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.197 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.210 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:01.210 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:01.210 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=2, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=15), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1837364347(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@142f293 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a5259b5 -> wakeup thread Time-limited test)) after lookup
18:21:01.211 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:01.212 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.212 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.213 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.230 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:01.230 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:01.231 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=4, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [18ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1837364347(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@142f293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a5259b5 -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787))) after lookup
18:21:01.231 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:01.232 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.232 [DEBUG - kudu-nio-2] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.233 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.270 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:01.270 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:01.271 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=6, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=75), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [18ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [36ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [38ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [38ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [75ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1837364347(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@142f293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a5259b5 -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914))) after lookup
18:21:01.271 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:01.272 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.273 [DEBUG - kudu-nio-3] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.273 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.310 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:01.310 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:01.311 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=8, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=115), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [18ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [36ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [38ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [38ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [75ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [76ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [78ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [78ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [111ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [115ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1837364347(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@142f293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914) -> (continuation of Deferred@932960202 after retry RPC@133645442), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a5259b5 -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914) -> (continuation of Deferred@932960202 after retry RPC@133645442))) after lookup
18:21:01.311 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:01.312 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.313 [DEBUG - kudu-nio-4] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.313 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.810 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:01.810 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:01.810 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=10, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=615), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [18ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [36ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [38ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [38ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [75ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [76ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [78ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [78ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [111ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [115ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [116ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [118ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [118ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [612ms] refreshing cache from master, [612ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [615ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1837364347(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@142f293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914) -> (continuation of Deferred@932960202 after retry RPC@133645442) -> (continuation of Deferred@1378959808 after retry RPC@1876941904), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a5259b5 -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914) -> (continuation of Deferred@932960202 after retry RPC@133645442) -> (continuation of Deferred@1378959808 after retry RPC@1876941904))) after lookup
18:21:01.811 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:01.812 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.812 [DEBUG - kudu-nio-5] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.813 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.930 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:01.931 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:01.932 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=12, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=736), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [18ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [36ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [38ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [38ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [75ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [76ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [78ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [78ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [111ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [115ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [116ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [118ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [118ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [612ms] refreshing cache from master, [612ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [615ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [616ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [618ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [618ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [732ms] refreshing cache from master, [732ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [735ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1837364347(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@142f293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914) -> (continuation of Deferred@932960202 after retry RPC@133645442) -> (continuation of Deferred@1378959808 after retry RPC@1876941904) -> (continuation of Deferred@1948358883 after retry RPC@1695414123), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a5259b5 -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914) -> (continuation of Deferred@932960202 after retry RPC@133645442) -> (continuation of Deferred@1378959808 after retry RPC@1876941904) -> (continuation of Deferred@1948358883 after retry RPC@1695414123))) after lookup
18:21:01.932 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:01.933 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.933 [DEBUG - kudu-nio-6] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:01.934 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.048528 21966 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:21:02.270 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:02.271 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:02.271 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=14, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1076), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [18ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [36ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [38ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [38ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [75ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [76ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [78ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [78ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [111ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [115ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [116ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [118ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [118ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [612ms] refreshing cache from master, [612ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [615ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [616ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [618ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [618ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [732ms] refreshing cache from master, [732ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [735ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [737ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [738ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [739ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [1072ms] refreshing cache from master, [1072ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1075ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1837364347(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@142f293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914) -> (continuation of Deferred@932960202 after retry RPC@133645442) -> (continuation of Deferred@1378959808 after retry RPC@1876941904) -> (continuation of Deferred@1948358883 after retry RPC@1695414123) -> (continuation of Deferred@1018221959 after retry RPC@906812373), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a5259b5 -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914) -> (continuation of Deferred@932960202 after retry RPC@133645442) -> (continuation of Deferred@1378959808 after retry RPC@1876941904) -> (continuation of Deferred@1948358883 after retry RPC@1695414123) -> (continuation of Deferred@1018221959 after retry RPC@906812373))) after lookup
18:21:02.272 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:02.273 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:02.274 [DEBUG - kudu-nio-7] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:02.274 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.447645 21985 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 4 FOLLOWER]: Starting pre-election (detected failure of leader b5a72ae51147432e815137a6da2c75ff)
18:21:02.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.447841 21985 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.449491 21985 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:02.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:02.450623 21623 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:02.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:02.452603 21623 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:02.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.457861 21912 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "90f4362b094e4d1681153e2008083f4e" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" is_pre_election: true
18:21:02.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.458403 21912 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 90f4362b094e4d1681153e2008083f4e in term 3.
18:21:02.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.459439 21621 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, afe5bf8ec6e44b15bad84f94f3ee7158; no voters: b5a72ae51147432e815137a6da2c75ff
18:21:02.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.459918 21985 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 4 FOLLOWER]: Leader pre-election won for term 5
18:21:02.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.460042 21985 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 4 FOLLOWER]: Starting leader election (detected failure of leader b5a72ae51147432e815137a6da2c75ff)
18:21:02.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.460107 21985 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 4 FOLLOWER]: Advancing to term 5
18:21:02.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.462077 21985 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:02.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.462698 21985 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 5 election: Requested vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:02.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.463254 21912 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "90f4362b094e4d1681153e2008083f4e" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158"
18:21:02.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.463466 21912 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 3 FOLLOWER]: Advancing to term 5
18:21:02.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:02.464076 21623 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:02.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.467465 21912 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 90f4362b094e4d1681153e2008083f4e in term 5.
18:21:02.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.468007 21621 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, afe5bf8ec6e44b15bad84f94f3ee7158; no voters: b5a72ae51147432e815137a6da2c75ff
18:21:02.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.468375 21985 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 5 FOLLOWER]: Leader election won for term 5
18:21:02.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.468892 21985 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 5 LEADER]: Becoming Leader. State: Replica: 90f4362b094e4d1681153e2008083f4e, State: Running, Role: LEADER
18:21:02.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.469499 21985 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:02.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.474316 20699 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e reported cstate change: term changed from 4 to 5, leader changed from b5a72ae51147432e815137a6da2c75ff (127.20.1.67) to 90f4362b094e4d1681153e2008083f4e (127.20.1.65). New cstate: current_term: 5 leader_uuid: "90f4362b094e4d1681153e2008083f4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: HEALTHY } } }
18:21:02.670 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:02.670 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:02.671 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=16, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=1475), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [12ms] refreshing cache from master, [12ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [14ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [16ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [17ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [18ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [32ms] refreshing cache from master, [32ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [35ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [36ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [38ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [38ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [72ms] refreshing cache from master, [72ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [75ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [76ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [78ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [78ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [111ms] refreshing cache from master, [112ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [115ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [116ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [118ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [118ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [612ms] refreshing cache from master, [612ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [615ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [616ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [618ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [618ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [732ms] refreshing cache from master, [732ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [735ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [737ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [738ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [739ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [1072ms] refreshing cache from master, [1072ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1075ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [1077ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [1079ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [1079ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [1471ms] refreshing cache from master, [1472ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1475ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1837364347(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@142f293 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914) -> (continuation of Deferred@932960202 after retry RPC@133645442) -> (continuation of Deferred@1378959808 after retry RPC@1876941904) -> (continuation of Deferred@1948358883 after retry RPC@1695414123) -> (continuation of Deferred@1018221959 after retry RPC@906812373) -> (continuation of Deferred@15316131 after retry RPC@1357579544), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4a5259b5 -> wakeup thread Time-limited test -> (continuation of Deferred@634963366 after retry RPC@2133186787) -> (continuation of Deferred@2132862005 after retry RPC@773052914) -> (continuation of Deferred@932960202 after retry RPC@133645442) -> (continuation of Deferred@1378959808 after retry RPC@1876941904) -> (continuation of Deferred@1948358883 after retry RPC@1695414123) -> (continuation of Deferred@1018221959 after retry RPC@906812373) -> (continuation of Deferred@15316131 after retry RPC@1357579544))) after lookup
18:21:02.672 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:02.673 [DEBUG - kudu-nio-0] (Connection.java:757) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] Successfully connected to peer
18:21:02.689 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x64876dac, L:/127.0.0.1:42404 - R:/127.20.1.65:36397] using TOKEN/null
18:21:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:02.695911 21623 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e -> Peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Couldn't send request to peer b5a72ae51147432e815137a6da2c75ff. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.695911 21912 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 5 FOLLOWER]: Refusing update from remote peer 90f4362b094e4d1681153e2008083f4e: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 18. (index mismatch)
18:21:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.696877 21991 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
18:21:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.707619 22007 mvcc.cc:204] Tried to move back new op lower bound from 7317391207191879680 to 7317391206281035776. Current Snapshot: MvccSnapshot[applied={T|T < 7317391197927260160 or (T in {7317391197957873664})}]
18:21:02.728 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.1.67:41125
18:21:02.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:02.729734 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/logs
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.67:41125
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.67
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46629
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.67
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:02.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:03.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:03.094553 22015 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:03.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:03.094880 22015 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:03.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:03.094959 22015 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:03.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:03.103669 22015 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:03.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:03.103871 22015 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.67
18:21:03.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.114476 22015 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/logs
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.67:41125
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.67
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46629
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.67
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:03.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:03.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:21:03.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:21:03.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:21:03.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:21:03.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:21:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.116726 22015 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:03.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.119375 22015 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:21:03.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:03.130617 22021 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:03.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:03.130700 22020 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:03.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:03.132027 22024 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:03.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.132406 22015 server_base.cc:1061] running on GCE node
18:21:03.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.133010 22015 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:03.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.134012 22015 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:03.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.135232 22015 hybrid_clock.cc:648] HybridClock initialized: now 1786472463135167 us; error 59 us; skew 500 ppm
18:21:03.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.135543 22015 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:21:03.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.137950 22015 webserver.cc:533] Webserver started at http://127.20.1.67:46629/ using document root <none> and password file <none>
18:21:03.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.138783 22015 fs_manager.cc:362] Metadata directory not provided
18:21:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.138898 22015 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.145318 22015 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.001s
18:21:03.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.149245 22031 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:03.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.151305 22015 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
18:21:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.151484 22015 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5a72ae51147432e815137a6da2c75ff"
18:21:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:50 on dist-test-slave-6k22"
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.152733 22015 fs_report.cc:389] FS layout report
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/data
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.184278 22015 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:03.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.185202 22015 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:03.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.185473 22015 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:03.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.186444 22015 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.188874 22038 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:21:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.195156 22015 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.195246 22015 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
18:21:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.195344 22015 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:03.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.198261 22015 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:03.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.198372 22015 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.001s
18:21:03.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.198954 22038 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap starting.
18:21:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.230780 22038 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Log is configured to *not* fsync() on all Append() calls
18:21:03.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.257782 22038 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:03.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.258672 22038 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap complete.
18:21:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.260586 22038 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Time spent bootstrapping tablet: real 0.062s	user 0.053s	sys 0.007s
18:21:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.269629 22015 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.67:41125
18:21:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.269682 22162 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.67:41125 every 8 connection(s)
18:21:03.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.272214 22015 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:21:03.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.271488 22038 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:03.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.272720 22038 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: b5a72ae51147432e815137a6da2c75ff, State: Initialized, Role: FOLLOWER
18:21:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.274777 22038 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:03.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.282616 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 22015
18:21:03.283 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.315496 22038 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Time spent starting tablet: real 0.055s	user 0.024s	sys 0.028s
18:21:03.320 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:03.320 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.1.65:36397
18:21:03.321 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:21:03.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.321641 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 21603
18:21:03.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.333648 22163 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:21:03.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.333751 22164 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:21:03.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.334156 22163 heartbeater.cc:461] Registering TS with master...
18:21:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.334161 22164 heartbeater.cc:461] Registering TS with master...
18:21:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.335151 22165 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:21:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.335668 22165 heartbeater.cc:461] Registering TS with master...
18:21:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.337373 22164 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:21:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.337383 22165 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:21:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.341352 20534 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:21:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.341647 20613 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:21:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.346802 22163 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:21:03.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.348394 20699 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:21:03.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.350226 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.67:50185
18:21:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.353523 22163 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:21:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.356420 21750 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:03.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:03.461764 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 21603
18:21:03.492 [DEBUG - kudu-nio-0] (Connection.java:239) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] handling channelInactive
18:21:03.492 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:03.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state READY due to: connection closed
18:21:03.493 [DEBUG - Time-limited test] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:03.494 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.494 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.494 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.510 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:03.510 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:03.510 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [17ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1957048818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1e858f95 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3622ead9 -> wakeup thread Time-limited test)) after lookup
18:21:03.511 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:03.512 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.512 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.512 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.530 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:03.530 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:03.531 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [17ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [19ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [20ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [20ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1957048818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1e858f95 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956652607 after retry RPC@407903716), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3622ead9 -> wakeup thread Time-limited test -> (continuation of Deferred@956652607 after retry RPC@407903716))) after lookup
18:21:03.531 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:03.532 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.532 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.533 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.571 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:03.571 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:03.571 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [17ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [19ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [20ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [20ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [39ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [41ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [41ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [78ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1957048818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1e858f95 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956652607 after retry RPC@407903716) -> (continuation of Deferred@711362606 after retry RPC@851517371), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3622ead9 -> wakeup thread Time-limited test -> (continuation of Deferred@956652607 after retry RPC@407903716) -> (continuation of Deferred@711362606 after retry RPC@851517371))) after lookup
18:21:03.572 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:03.573 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.573 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.574 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.650 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:03.651 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:03.651 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=159), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [17ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [19ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [20ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [20ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [39ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [41ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [41ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [78ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [80ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [82ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [82ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [155ms] refreshing cache from master, [155ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [158ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1957048818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1e858f95 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956652607 after retry RPC@407903716) -> (continuation of Deferred@711362606 after retry RPC@851517371) -> (continuation of Deferred@545932324 after retry RPC@437037897), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3622ead9 -> wakeup thread Time-limited test -> (continuation of Deferred@956652607 after retry RPC@407903716) -> (continuation of Deferred@711362606 after retry RPC@851517371) -> (continuation of Deferred@545932324 after retry RPC@437037897))) after lookup
18:21:03.652 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:03.653 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.653 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.654 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.950 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:03.950 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:03.951 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=458), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [17ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [19ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [20ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [20ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [39ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [41ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [41ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [78ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [80ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [82ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [82ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [155ms] refreshing cache from master, [155ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [158ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [160ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [162ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [162ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [455ms] refreshing cache from master, [455ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [458ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1957048818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1e858f95 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956652607 after retry RPC@407903716) -> (continuation of Deferred@711362606 after retry RPC@851517371) -> (continuation of Deferred@545932324 after retry RPC@437037897) -> (continuation of Deferred@263590968 after retry RPC@1889513572), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3622ead9 -> wakeup thread Time-limited test -> (continuation of Deferred@956652607 after retry RPC@407903716) -> (continuation of Deferred@711362606 after retry RPC@851517371) -> (continuation of Deferred@545932324 after retry RPC@437037897) -> (continuation of Deferred@263590968 after retry RPC@1889513572))) after lookup
18:21:03.952 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:03.953 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.953 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:03.953 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.769518 22178 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader 90f4362b094e4d1681153e2008083f4e)
18:21:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.769739 22178 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.771337 22178 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:04.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:04.773171 21829 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111)
18:21:04.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:04.775293 21829 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397): Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111)
18:21:04.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.780344 22111 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "b5a72ae51147432e815137a6da2c75ff" is_pre_election: true
18:21:04.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.780844 22111 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate afe5bf8ec6e44b15bad84f94f3ee7158 in term 4.
18:21:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.781754 21829 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: afe5bf8ec6e44b15bad84f94f3ee7158, b5a72ae51147432e815137a6da2c75ff; no voters: 90f4362b094e4d1681153e2008083f4e
18:21:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.782181 22178 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 5 FOLLOWER]: Leader pre-election won for term 6
18:21:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.782295 22178 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 5 FOLLOWER]: Starting leader election (detected failure of leader 90f4362b094e4d1681153e2008083f4e)
18:21:04.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.782362 22178 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 5 FOLLOWER]: Advancing to term 6
18:21:04.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.784515 22178 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:04.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.785035 22178 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 6 election: Requested vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:04.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.785612 22111 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "b5a72ae51147432e815137a6da2c75ff"
18:21:04.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.785822 22111 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 4 FOLLOWER]: Advancing to term 6
18:21:04.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:04.786681 21829 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397): Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111)
18:21:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.789958 22111 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate afe5bf8ec6e44b15bad84f94f3ee7158 in term 6.
18:21:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.790458 21829 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: afe5bf8ec6e44b15bad84f94f3ee7158, b5a72ae51147432e815137a6da2c75ff; no voters: 90f4362b094e4d1681153e2008083f4e
18:21:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.790798 22178 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 6 FOLLOWER]: Leader election won for term 6
18:21:04.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.791266 22178 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 6 LEADER]: Becoming Leader. State: Replica: afe5bf8ec6e44b15bad84f94f3ee7158, State: Running, Role: LEADER
18:21:04.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.792109 22178 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:04.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:04.796182 20699 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 reported cstate change: term changed from 5 to 6, leader changed from 90f4362b094e4d1681153e2008083f4e (127.20.1.65) to afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66). New cstate: current_term: 6 leader_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: UNKNOWN } } }
18:21:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:05.309711 21829 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 -> Peer 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397): Couldn't send request to peer 90f4362b094e4d1681153e2008083f4e. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:05.333969 22111 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 6 FOLLOWER]: Refusing update from remote peer afe5bf8ec6e44b15bad84f94f3ee7158: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
18:21:05.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:05.334935 22198 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
18:21:05.790 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:05.791 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:05.791 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2299), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [2ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [15ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [17ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [19ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [20ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [20ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [34ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [39ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [41ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [41ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [75ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [78ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [80ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [82ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [82ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [155ms] refreshing cache from master, [155ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [158ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [160ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [162ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [162ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [455ms] refreshing cache from master, [455ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [458ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [459ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [461ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [461ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [2295ms] refreshing cache from master, [2295ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [2298ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1957048818(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1e858f95 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@956652607 after retry RPC@407903716) -> (continuation of Deferred@711362606 after retry RPC@851517371) -> (continuation of Deferred@545932324 after retry RPC@437037897) -> (continuation of Deferred@263590968 after retry RPC@1889513572) -> (continuation of Deferred@1783471908 after retry RPC@1734086297), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@3622ead9 -> wakeup thread Time-limited test -> (continuation of Deferred@956652607 after retry RPC@407903716) -> (continuation of Deferred@711362606 after retry RPC@851517371) -> (continuation of Deferred@545932324 after retry RPC@437037897) -> (continuation of Deferred@263590968 after retry RPC@1889513572) -> (continuation of Deferred@1783471908 after retry RPC@1734086297))) after lookup
18:21:05.792 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:05.793 [DEBUG - kudu-nio-7] (Connection.java:757) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] Successfully connected to peer
18:21:05.812 [DEBUG - kudu-nio-7] (Negotiator.java:918) Authenticated connection [id: 0x7b5f05e4, L:/127.0.0.1:50314 - R:/127.20.1.66:40881] using TOKEN/null
18:21:05.838 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.1.65:36397
18:21:05.840 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:05.841406 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data
18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/logs
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.65:36397
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.65
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46555
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.65
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:06.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:06.204123 22216 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:06.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:06.204487 22216 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:06.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:06.204599 22216 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:06.213429 22216 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:06.213639 22216 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.65
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.223737 22216 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/logs
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.65:36397
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.65
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46555
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.65
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:21:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:21:06.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.225971 22216 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:06.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.228082 22216 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:21:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:06.238162 22222 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:06.238175 22221 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:06.239058 22224 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.239421 22216 server_base.cc:1061] running on GCE node
18:21:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.240041 22216 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:06.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.241088 22216 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:06.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.242343 22216 hybrid_clock.cc:648] HybridClock initialized: now 1786472466242289 us; error 61 us; skew 500 ppm
18:21:06.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.242693 22216 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:21:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.244966 22216 webserver.cc:533] Webserver started at http://127.20.1.65:46555/ using document root <none> and password file <none>
18:21:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.245724 22216 fs_manager.cc:362] Metadata directory not provided
18:21:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.245867 22216 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.251214 22216 fs_manager.cc:714] Time spent opening directory manager: real 0.003s	user 0.002s	sys 0.001s
18:21:06.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.254305 22230 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:06.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.255668 22216 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.002s
18:21:06.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.255841 22216 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:21:06.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90f4362b094e4d1681153e2008083f4e"
18:21:06.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:21:06.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.256665 22216 fs_report.cc:389] FS layout report
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/data
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:06.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.284770 22216 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:06.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.285743 22216 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:06.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.286103 22216 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:06.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.287168 22216 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.289561 22237 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:21:06.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.296025 22216 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:06.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.296145 22216 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
18:21:06.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.296300 22216 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:06.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.299383 22216 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:06.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.299515 22216 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
18:21:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.300060 22237 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Bootstrap starting.
18:21:06.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.329919 22237 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Log is configured to *not* fsync() on all Append() calls
18:21:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.371621 22216 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.65:36397
18:21:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.371687 22362 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.65:36397 every 8 connection(s)
18:21:06.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.373262 22237 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:06.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.373775 22216 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:21:06.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.374159 22237 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Bootstrap complete.
18:21:06.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.376442 22237 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Time spent bootstrapping tablet: real 0.077s	user 0.051s	sys 0.023s
18:21:06.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.377707 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 22216
18:21:06.382 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:06.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.387516 22237 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.389678 22237 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: 90f4362b094e4d1681153e2008083f4e, State: Initialized, Role: FOLLOWER
18:21:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.390753 22237 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:06.438 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:06.439 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.1.66:40881
18:21:06.439 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.440061 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 21808
18:21:06.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.448103 22237 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Time spent starting tablet: real 0.072s	user 0.026s	sys 0.034s
18:21:06.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.457655 22365 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:21:06.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.458128 22365 heartbeater.cc:461] Registering TS with master...
18:21:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.459276 22365 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:21:06.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.463337 20613 ts_manager.cc:194] Re-registered known tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:06.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.465610 22364 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:21:06.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.465777 22364 heartbeater.cc:461] Registering TS with master...
18:21:06.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.466346 22364 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:21:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.467958 20534 ts_manager.cc:194] Re-registered known tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.470295 22363 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:21:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.470439 22363 heartbeater.cc:461] Registering TS with master...
18:21:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.470810 22363 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:21:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.472043 20699 ts_manager.cc:194] Re-registered known tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.473728 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.65:49965
18:21:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.475284 21959 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:06.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.477029 22363 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:21:06.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:06.579421 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 21808
18:21:06.607 [DEBUG - kudu-nio-7] (Connection.java:239) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] handling channelInactive
18:21:06.607 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:06.608 [DEBUG - kudu-nio-7] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state READY due to: connection closed
18:21:06.608 [DEBUG - Time-limited test] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:06.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.610 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.630 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:06.630 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:06.630 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=2, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=22), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@923630767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@30229411 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c8e8296 -> wakeup thread Time-limited test)) after lookup
18:21:06.631 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:06.632 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.632 [DEBUG - kudu-nio-1] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.632 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.650 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:06.650 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:06.651 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=4, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=42), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [23ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [24ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [24ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@923630767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@30229411 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1518596781 after retry RPC@1030724364), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c8e8296 -> wakeup thread Time-limited test -> (continuation of Deferred@1518596781 after retry RPC@1030724364))) after lookup
18:21:06.651 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:06.652 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.653 [DEBUG - kudu-nio-2] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.653 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.691 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:06.691 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:06.692 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=6, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=83), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [23ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [24ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [24ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [43ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [45ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [45ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [82ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@923630767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@30229411 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1518596781 after retry RPC@1030724364) -> (continuation of Deferred@1426646481 after retry RPC@343843719), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c8e8296 -> wakeup thread Time-limited test -> (continuation of Deferred@1518596781 after retry RPC@1030724364) -> (continuation of Deferred@1426646481 after retry RPC@343843719))) after lookup
18:21:06.692 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:06.693 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.693 [DEBUG - kudu-nio-3] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.693 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.750 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:06.751 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:06.751 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=8, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=143), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [23ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [24ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [24ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [43ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [45ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [45ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [82ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [84ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [85ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [85ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [139ms] refreshing cache from master, [139ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [142ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@923630767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@30229411 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1518596781 after retry RPC@1030724364) -> (continuation of Deferred@1426646481 after retry RPC@343843719) -> (continuation of Deferred@2035517844 after retry RPC@407924642), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c8e8296 -> wakeup thread Time-limited test -> (continuation of Deferred@1518596781 after retry RPC@1030724364) -> (continuation of Deferred@1426646481 after retry RPC@343843719) -> (continuation of Deferred@2035517844 after retry RPC@407924642))) after lookup
18:21:06.752 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:06.753 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.753 [DEBUG - kudu-nio-4] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:06.753 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:07.050 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:07.050 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:07.050 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=10, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=442), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [23ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [24ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [24ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [43ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [45ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [45ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [82ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [84ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [85ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [85ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [139ms] refreshing cache from master, [139ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [142ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [143ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [145ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [145ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [438ms] refreshing cache from master, [438ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [441ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@923630767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@30229411 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1518596781 after retry RPC@1030724364) -> (continuation of Deferred@1426646481 after retry RPC@343843719) -> (continuation of Deferred@2035517844 after retry RPC@407924642) -> (continuation of Deferred@1313430428 after retry RPC@1434386568), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c8e8296 -> wakeup thread Time-limited test -> (continuation of Deferred@1518596781 after retry RPC@1030724364) -> (continuation of Deferred@1426646481 after retry RPC@343843719) -> (continuation of Deferred@2035517844 after retry RPC@407924642) -> (continuation of Deferred@1313430428 after retry RPC@1434386568))) after lookup
18:21:07.051 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:07.052 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:07.052 [DEBUG - kudu-nio-5] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:07.052 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:07.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.968649 22377 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 6 FOLLOWER]: Starting pre-election (detected failure of leader afe5bf8ec6e44b15bad84f94f3ee7158)
18:21:07.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.968917 22377 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:07.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.970623 22377 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:07.971941 22032 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:07.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:07.973649 22032 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881): Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.979384 22312 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "b5a72ae51147432e815137a6da2c75ff" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e" is_pre_election: true
18:21:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.979882 22312 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b5a72ae51147432e815137a6da2c75ff in term 5.
18:21:07.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.980918 22034 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, b5a72ae51147432e815137a6da2c75ff; no voters: afe5bf8ec6e44b15bad84f94f3ee7158
18:21:07.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.981285 22377 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 6 FOLLOWER]: Leader pre-election won for term 7
18:21:07.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.981431 22377 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 6 FOLLOWER]: Starting leader election (detected failure of leader afe5bf8ec6e44b15bad84f94f3ee7158)
18:21:07.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.981544 22377 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 6 FOLLOWER]: Advancing to term 7
18:21:07.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.983635 22377 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:07.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.984310 22377 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 7 election: Requested vote from peers afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.984898 22312 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "b5a72ae51147432e815137a6da2c75ff" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e"
18:21:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.985105 22312 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 5 FOLLOWER]: Advancing to term 7
18:21:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:07.986006 22032 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881): Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:07.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.989180 22312 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b5a72ae51147432e815137a6da2c75ff in term 7.
18:21:07.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.989673 22034 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, b5a72ae51147432e815137a6da2c75ff; no voters: afe5bf8ec6e44b15bad84f94f3ee7158
18:21:07.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.989992 22377 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 7 FOLLOWER]: Leader election won for term 7
18:21:07.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.990461 22377 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 7 LEADER]: Becoming Leader. State: Replica: b5a72ae51147432e815137a6da2c75ff, State: Running, Role: LEADER
18:21:07.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.990957 22377 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:07.995674 20699 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff reported cstate change: term changed from 6 to 7, leader changed from afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66) to b5a72ae51147432e815137a6da2c75ff (127.20.1.67). New cstate: current_term: 7 leader_uuid: "b5a72ae51147432e815137a6da2c75ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: UNKNOWN } } }
18:21:08.430 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:08.431 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:08.431 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=12, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=1823), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [2ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [19ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [23ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [24ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [24ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [38ms] refreshing cache from master, [39ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [41ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [43ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [45ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [45ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [79ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [82ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [84ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [85ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [85ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [139ms] refreshing cache from master, [139ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [142ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [143ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [145ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [145ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [438ms] refreshing cache from master, [438ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [441ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [443ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [444ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [444ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [1819ms] refreshing cache from master, [1819ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1822ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@923630767(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@30229411 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1518596781 after retry RPC@1030724364) -> (continuation of Deferred@1426646481 after retry RPC@343843719) -> (continuation of Deferred@2035517844 after retry RPC@407924642) -> (continuation of Deferred@1313430428 after retry RPC@1434386568) -> (continuation of Deferred@1205554353 after retry RPC@1779265410), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@4c8e8296 -> wakeup thread Time-limited test -> (continuation of Deferred@1518596781 after retry RPC@1030724364) -> (continuation of Deferred@1426646481 after retry RPC@343843719) -> (continuation of Deferred@2035517844 after retry RPC@407924642) -> (continuation of Deferred@1313430428 after retry RPC@1434386568) -> (continuation of Deferred@1205554353 after retry RPC@1779265410))) after lookup
18:21:08.432 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:08.433 [DEBUG - kudu-nio-6] (Connection.java:757) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] Successfully connected to peer
18:21:08.458 [DEBUG - kudu-nio-6] (Negotiator.java:918) Authenticated connection [id: 0x0139709d, L:/127.0.0.1:54684 - R:/127.20.1.67:41125] using TOKEN/null
18:21:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.464512 22312 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 7 FOLLOWER]: Refusing update from remote peer b5a72ae51147432e815137a6da2c75ff: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 26. (index mismatch)
18:21:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:08.464651 22032 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff -> Peer afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881): Couldn't send request to peer afe5bf8ec6e44b15bad84f94f3ee7158. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.465516 22377 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [LEADER]: Connected to new peer: Peer: permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
18:21:08.504 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.1.66:40881
18:21:08.505 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.505888 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/logs
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.66:40881
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.66
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40147
18:21:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.66
18:21:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:08.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:08.875108 22406 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:08.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:08.875536 22406 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:08.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:08.875694 22406 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:08.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:08.884735 22406 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:08.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:08.884979 22406 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.66
18:21:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.895246 22406 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/logs
18:21:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:08.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.66:40881
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.66
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40147
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.66
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:21:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:21:08.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.897490 22406 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.899784 22406 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:21:08.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:08.910065 22411 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:08.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:08.910070 22412 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:08.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:08.910636 22414 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:08.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.911880 22406 server_base.cc:1061] running on GCE node
18:21:08.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.912539 22406 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:08.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.913595 22406 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:08.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.914847 22406 hybrid_clock.cc:648] HybridClock initialized: now 1786472468914789 us; error 41 us; skew 500 ppm
18:21:08.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.915242 22406 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:21:08.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.917793 22406 webserver.cc:533] Webserver started at http://127.20.1.66:40147/ using document root <none> and password file <none>
18:21:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.918653 22406 fs_manager.cc:362] Metadata directory not provided
18:21:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.918797 22406 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.924665 22406 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.004s	sys 0.000s
18:21:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.928580 22420 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.930043 22406 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.930245 22406 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afe5bf8ec6e44b15bad84f94f3ee7158"
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.931054 22406 fs_report.cc:389] FS layout report
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/data
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.948693 22406 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.949620 22406 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.949982 22406 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.950986 22406 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.953197 22427 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:21:08.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.959729 22406 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:08.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.959884 22406 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s	user 0.001s	sys 0.000s
18:21:08.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.960028 22406 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.963214 22406 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.963357 22406 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.000s	sys 0.000s
18:21:08.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:08.963881 22427 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Bootstrap starting.
18:21:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.006551 22427 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Log is configured to *not* fsync() on all Append() calls
18:21:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.044258 22427 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:09.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.045995 22427 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Bootstrap complete.
18:21:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.047992 22427 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Time spent bootstrapping tablet: real 0.085s	user 0.046s	sys 0.035s
18:21:09.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.053213 22427 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:09.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.054342 22427 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: afe5bf8ec6e44b15bad84f94f3ee7158, State: Initialized, Role: FOLLOWER
18:21:09.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.054708 22406 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.66:40881
18:21:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.054783 22551 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.66:40881 every 8 connection(s)
18:21:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.055383 22427 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:09.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.056725 22406 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:21:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.057077 22427 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Time spent starting tablet: real 0.009s	user 0.008s	sys 0.000s
18:21:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.059353 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 22406
18:21:09.060 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:09.086 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:09.087 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.1.67:41125
18:21:09.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:09.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.088722 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 22015
18:21:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.090857 22555 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:21:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.091364 22555 heartbeater.cc:461] Registering TS with master...
18:21:09.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.092762 22555 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:21:09.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.095621 22553 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:21:09.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.095781 22553 heartbeater.cc:461] Registering TS with master...
18:21:09.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.096364 22553 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:21:09.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.097035 20613 ts_manager.cc:194] Re-registered known tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:09.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.098127 20699 ts_manager.cc:194] Re-registered known tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:09.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.099488 22554 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:21:09.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.099628 22554 heartbeater.cc:461] Registering TS with master...
18:21:09.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.099972 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.66:52035
18:21:09.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.100081 22554 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:21:09.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.101608 20534 ts_manager.cc:194] Re-registered known tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.109959 22158 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:09.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:09.210520 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 22015
18:21:09.238 [DEBUG - kudu-nio-6] (Connection.java:239) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] handling channelInactive
18:21:09.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:09.239 [DEBUG - kudu-nio-6] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state READY due to: connection closed
18:21:09.239 [DEBUG - Time-limited test] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:09.240 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.241 [DEBUG - kudu-nio-7] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.241 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.250 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:09.250 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:09.250 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=11), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1320005582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4fddb8e6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1245843a -> wakeup thread Time-limited test)) after lookup
18:21:09.251 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:09.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.252 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.270 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:09.271 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:09.271 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=32), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [11ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [13ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [13ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1320005582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4fddb8e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@920155046 after retry RPC@134370579), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1245843a -> wakeup thread Time-limited test -> (continuation of Deferred@920155046 after retry RPC@134370579))) after lookup
18:21:09.271 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:09.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.272 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.310 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:09.311 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:09.311 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=72), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [11ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [13ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [13ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [32ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [33ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [33ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1320005582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4fddb8e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@920155046 after retry RPC@134370579) -> (continuation of Deferred@116185463 after retry RPC@507901659), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1245843a -> wakeup thread Time-limited test -> (continuation of Deferred@920155046 after retry RPC@134370579) -> (continuation of Deferred@116185463 after retry RPC@507901659))) after lookup
18:21:09.312 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:09.313 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.313 [DEBUG - kudu-nio-2] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.313 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.430 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:09.431 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:09.431 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=192), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [11ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [13ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [13ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [32ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [33ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [33ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [72ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [74ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [74ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [187ms] refreshing cache from master, [188ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [191ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1320005582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4fddb8e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@920155046 after retry RPC@134370579) -> (continuation of Deferred@116185463 after retry RPC@507901659) -> (continuation of Deferred@1741410614 after retry RPC@1451823162), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1245843a -> wakeup thread Time-limited test -> (continuation of Deferred@920155046 after retry RPC@134370579) -> (continuation of Deferred@116185463 after retry RPC@507901659) -> (continuation of Deferred@1741410614 after retry RPC@1451823162))) after lookup
18:21:09.432 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:09.433 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.433 [DEBUG - kudu-nio-3] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.433 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.730 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:09.730 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:09.731 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=492), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [11ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [13ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [13ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [32ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [33ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [33ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [72ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [74ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [74ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [187ms] refreshing cache from master, [188ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [191ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [192ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [194ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [194ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [487ms] refreshing cache from master, [488ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [491ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1320005582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4fddb8e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@920155046 after retry RPC@134370579) -> (continuation of Deferred@116185463 after retry RPC@507901659) -> (continuation of Deferred@1741410614 after retry RPC@1451823162) -> (continuation of Deferred@1369800425 after retry RPC@1337179175), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1245843a -> wakeup thread Time-limited test -> (continuation of Deferred@920155046 after retry RPC@134370579) -> (continuation of Deferred@116185463 after retry RPC@507901659) -> (continuation of Deferred@1741410614 after retry RPC@1451823162) -> (continuation of Deferred@1369800425 after retry RPC@1337179175))) after lookup
18:21:09.731 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:09.732 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.732 [DEBUG - kudu-nio-4] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:09.732 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.104537 22553 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:21:10.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.213892 22567 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 6 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:21:10.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.214301 22567 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.216341 22567 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:10.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:10.218072 22423 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:10.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:10.220112 22423 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.224025 22312 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e" is_pre_election: true
18:21:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.224994 22423 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: afe5bf8ec6e44b15bad84f94f3ee7158; no voters: 90f4362b094e4d1681153e2008083f4e, b5a72ae51147432e815137a6da2c75ff
18:21:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.225379 22567 raft_consensus.cc:2749] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 6 FOLLOWER]: Leader pre-election lost for term 7. Reason: could not achieve majority
18:21:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.545281 22572 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 7 FOLLOWER]: Starting pre-election (detected failure of leader b5a72ae51147432e815137a6da2c75ff)
18:21:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.545499 22572 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.547132 22572 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:10.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:10.548663 22233 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:10.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:10.550681 22233 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.555966 22501 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "90f4362b094e4d1681153e2008083f4e" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" is_pre_election: true
18:21:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.556362 22501 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 90f4362b094e4d1681153e2008083f4e in term 6.
18:21:10.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.557304 22231 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, afe5bf8ec6e44b15bad84f94f3ee7158; no voters: b5a72ae51147432e815137a6da2c75ff
18:21:10.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.557699 22572 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 7 FOLLOWER]: Leader pre-election won for term 8
18:21:10.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.557868 22572 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 7 FOLLOWER]: Starting leader election (detected failure of leader b5a72ae51147432e815137a6da2c75ff)
18:21:10.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.557998 22572 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 7 FOLLOWER]: Advancing to term 8
18:21:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.560243 22572 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.560982 22572 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 8 election: Requested vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.561600 22501 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "90f4362b094e4d1681153e2008083f4e" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158"
18:21:10.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.561888 22501 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 6 FOLLOWER]: Advancing to term 8
18:21:10.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:10.562423 22233 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:10.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.565979 22501 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 90f4362b094e4d1681153e2008083f4e in term 8.
18:21:10.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.566542 22231 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, afe5bf8ec6e44b15bad84f94f3ee7158; no voters: b5a72ae51147432e815137a6da2c75ff
18:21:10.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.566905 22572 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 8 FOLLOWER]: Leader election won for term 8
18:21:10.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.567368 22572 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 8 LEADER]: Becoming Leader. State: Replica: 90f4362b094e4d1681153e2008083f4e, State: Running, Role: LEADER
18:21:10.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.568158 22572 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.572023 20699 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e reported cstate change: term changed from 7 to 8, leader changed from b5a72ae51147432e815137a6da2c75ff (127.20.1.67) to 90f4362b094e4d1681153e2008083f4e (127.20.1.65). New cstate: current_term: 8 leader_uuid: "90f4362b094e4d1681153e2008083f4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: HEALTHY } } }
18:21:10.871 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:10.871 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:10.872 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1632), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [2ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [2ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [8ms] refreshing cache from master, [8ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [10ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [11ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [13ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [13ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [31ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [32ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [33ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [33ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [71ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [72ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [74ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [74ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [187ms] refreshing cache from master, [188ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [191ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [192ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [194ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [194ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [487ms] refreshing cache from master, [488ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [491ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [492ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [493ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [494ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [1628ms] refreshing cache from master, [1628ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1631ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1320005582(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4fddb8e6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@920155046 after retry RPC@134370579) -> (continuation of Deferred@116185463 after retry RPC@507901659) -> (continuation of Deferred@1741410614 after retry RPC@1451823162) -> (continuation of Deferred@1369800425 after retry RPC@1337179175) -> (continuation of Deferred@1598137901 after retry RPC@230779750), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1245843a -> wakeup thread Time-limited test -> (continuation of Deferred@920155046 after retry RPC@134370579) -> (continuation of Deferred@116185463 after retry RPC@507901659) -> (continuation of Deferred@1741410614 after retry RPC@1451823162) -> (continuation of Deferred@1369800425 after retry RPC@1337179175) -> (continuation of Deferred@1598137901 after retry RPC@230779750))) after lookup
18:21:10.872 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:10.873 [DEBUG - kudu-nio-5] (Connection.java:757) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] Successfully connected to peer
18:21:10.900 [DEBUG - kudu-nio-5] (Negotiator.java:918) Authenticated connection [id: 0xa6e90e2c, L:/127.0.0.1:35034 - R:/127.20.1.65:36397] using TOKEN/null
18:21:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.907282 22501 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 8 FOLLOWER]: Refusing update from remote peer 90f4362b094e4d1681153e2008083f4e: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 30. (index mismatch)
18:21:10.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:10.907449 22233 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e -> Peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Couldn't send request to peer b5a72ae51147432e815137a6da2c75ff. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:10.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.908324 22577 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
18:21:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.922911 22593 mvcc.cc:204] Tried to move back new op lower bound from 7317391240824365056 to 7317391239450955776. Current Snapshot: MvccSnapshot[applied={T|T < 7317391240824365056}]
18:21:10.941 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.1.67:41125
18:21:10.941 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:10.942528 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/logs
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:10.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.67:41125
18:21:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.67
18:21:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46629
18:21:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.67
18:21:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:11.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:11.297444 22603 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:11.297760 22603 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:11.297837 22603 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:11.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:11.306373 22603 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:11.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:11.306533 22603 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.67
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.316748 22603 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/logs
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.67:41125
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.67
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46629
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.67
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:21:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:21:11.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.318416 22603 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:11.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.320333 22603 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:21:11.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:11.330930 22610 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:11.330924 22609 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:11.336365 22612 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.337204 22603 server_base.cc:1061] running on GCE node
18:21:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.337834 22603 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.338832 22603 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.340099 22603 hybrid_clock.cc:648] HybridClock initialized: now 1786472471340030 us; error 51 us; skew 500 ppm
18:21:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.340497 22603 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:21:11.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.342990 22603 webserver.cc:533] Webserver started at http://127.20.1.67:46629/ using document root <none> and password file <none>
18:21:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.343925 22603 fs_manager.cc:362] Metadata directory not provided
18:21:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.344071 22603 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:11.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.349677 22603 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.001s	sys 0.004s
18:21:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.352806 22620 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:11.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.354321 22603 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.001s
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.354462 22603 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5a72ae51147432e815137a6da2c75ff"
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:50 on dist-test-slave-6k22"
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.355180 22603 fs_report.cc:389] FS layout report
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/data
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.375100 22603 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.376202 22603 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.376611 22603 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.377768 22603 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.380304 22628 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:21:11.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.388995 22603 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:11.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.389238 22603 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.009s	user 0.001s	sys 0.000s
18:21:11.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.389390 22603 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:11.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.392800 22603 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:11.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.392935 22603 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.003s	sys 0.000s
18:21:11.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.393476 22628 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap starting.
18:21:11.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.448199 22628 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Log is configured to *not* fsync() on all Append() calls
18:21:11.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.479346 22603 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.67:41125
18:21:11.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.479588 22748 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.67:41125 every 8 connection(s)
18:21:11.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.481703 22603 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:21:11.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.485522 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 22603
18:21:11.486 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.506557 22628 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:11.511 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:11.512 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.1.65:36397
18:21:11.512 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:21:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.513026 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 22216
18:21:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.524027 22628 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap complete.
18:21:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.527737 22628 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Time spent bootstrapping tablet: real 0.135s	user 0.058s	sys 0.045s
18:21:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.552050 22628 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:11.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.553875 22628 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: b5a72ae51147432e815137a6da2c75ff, State: Initialized, Role: FOLLOWER
18:21:11.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.556531 22628 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.557641 22756 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:21:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.558243 22756 heartbeater.cc:461] Registering TS with master...
18:21:11.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.560017 22756 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:21:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.565534 20699 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:21:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.566475 22357 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.568113 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.67:39975
18:21:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.575160 22760 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:21:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.575412 22760 heartbeater.cc:461] Registering TS with master...
18:21:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.576031 22760 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:21:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.578477 20534 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:21:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.588272 22756 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:21:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.589708 22628 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Time spent starting tablet: real 0.046s	user 0.014s	sys 0.003s
18:21:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.598657 22765 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:21:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.598842 22765 heartbeater.cc:461] Registering TS with master...
18:21:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.599206 22765 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:21:11.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.600657 20613 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:21:11.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:11.704730 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 22216
18:21:11.731 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:11.731 [DEBUG - kudu-nio-5] (Connection.java:239) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] handling channelInactive
18:21:11.731 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state READY due to: connection closed
18:21:11.747 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:11.749 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.749 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.749 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.770 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:11.770 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:11.770 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=3, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=39), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [1ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [18ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1209059719(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e8c11b8 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b10761d -> wakeup thread Time-limited test)) after lookup
18:21:11.771 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:11.772 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.772 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.772 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.791 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:11.791 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:11.791 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=5, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=59), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [1ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [18ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [40ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [41ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [41ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [59ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1209059719(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e8c11b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@476493971 after retry RPC@826245852), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b10761d -> wakeup thread Time-limited test -> (continuation of Deferred@476493971 after retry RPC@826245852))) after lookup
18:21:11.792 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:11.792 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.793 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.793 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.830 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:11.830 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:11.830 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=7, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=99), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [1ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [18ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [40ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [41ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [41ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [59ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [60ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [62ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [62ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [99ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1209059719(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e8c11b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@476493971 after retry RPC@826245852) -> (continuation of Deferred@1910744080 after retry RPC@1007153554), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b10761d -> wakeup thread Time-limited test -> (continuation of Deferred@476493971 after retry RPC@826245852) -> (continuation of Deferred@1910744080 after retry RPC@1007153554))) after lookup
18:21:11.831 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:11.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:11.832 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:12.050 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:12.050 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:12.051 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=9, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=319), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [1ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [18ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [40ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [41ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [41ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [59ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [60ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [62ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [62ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [99ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [100ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [101ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [101ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [316ms] refreshing cache from master, [316ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [319ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1209059719(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e8c11b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@476493971 after retry RPC@826245852) -> (continuation of Deferred@1910744080 after retry RPC@1007153554) -> (continuation of Deferred@910853312 after retry RPC@1255409336), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b10761d -> wakeup thread Time-limited test -> (continuation of Deferred@476493971 after retry RPC@826245852) -> (continuation of Deferred@1910744080 after retry RPC@1007153554) -> (continuation of Deferred@910853312 after retry RPC@1255409336))) after lookup
18:21:12.051 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:12.052 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:12.052 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:12.053 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:12.590 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:12.591 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)[L],afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:12.591 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=11, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=859), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [1ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [18ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [40ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [41ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [41ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [59ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [60ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [62ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [62ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [99ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [100ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [101ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [101ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [316ms] refreshing cache from master, [316ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [319ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [320ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [322ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [322ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [856ms] refreshing cache from master, [856ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [859ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1209059719(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e8c11b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@476493971 after retry RPC@826245852) -> (continuation of Deferred@1910744080 after retry RPC@1007153554) -> (continuation of Deferred@910853312 after retry RPC@1255409336) -> (continuation of Deferred@1819760563 after retry RPC@108821827), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b10761d -> wakeup thread Time-limited test -> (continuation of Deferred@476493971 after retry RPC@826245852) -> (continuation of Deferred@1910744080 after retry RPC@1007153554) -> (continuation of Deferred@910853312 after retry RPC@1255409336) -> (continuation of Deferred@1819760563 after retry RPC@108821827))) after lookup
18:21:12.592 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] connecting to peer
18:21:12.593 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:12.593 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] cleaning up while in state CONNECTING due to: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:12.593 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397
18:21:12.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.970189 22774 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader 90f4362b094e4d1681153e2008083f4e)
18:21:12.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.970394 22774 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:12.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.971027 22774 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:12.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:12.974488 22423 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397): Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111)
18:21:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.978194 22694 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "b5a72ae51147432e815137a6da2c75ff" is_pre_election: true
18:21:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.978664 22694 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate afe5bf8ec6e44b15bad84f94f3ee7158 in term 7.
18:21:12.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.979226 22423 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: afe5bf8ec6e44b15bad84f94f3ee7158, b5a72ae51147432e815137a6da2c75ff; no voters: 90f4362b094e4d1681153e2008083f4e
18:21:12.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.979609 22774 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 8 FOLLOWER]: Leader pre-election won for term 9
18:21:12.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.979723 22774 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 8 FOLLOWER]: Starting leader election (detected failure of leader 90f4362b094e4d1681153e2008083f4e)
18:21:12.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.979794 22774 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 8 FOLLOWER]: Advancing to term 9
18:21:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.981851 22774 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.982400 22774 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 9 election: Requested vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.982908 22694 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "b5a72ae51147432e815137a6da2c75ff"
18:21:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.983114 22694 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 7 FOLLOWER]: Advancing to term 9
18:21:12.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:12.984027 22423 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397): Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111)
18:21:12.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.987092 22694 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate afe5bf8ec6e44b15bad84f94f3ee7158 in term 9.
18:21:12.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.987632 22423 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: afe5bf8ec6e44b15bad84f94f3ee7158, b5a72ae51147432e815137a6da2c75ff; no voters: 90f4362b094e4d1681153e2008083f4e
18:21:12.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.987954 22774 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 9 FOLLOWER]: Leader election won for term 9
18:21:12.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.988422 22774 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 9 LEADER]: Becoming Leader. State: Replica: afe5bf8ec6e44b15bad84f94f3ee7158, State: Running, Role: LEADER
18:21:12.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.988966 22774 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:12.992576 20699 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 reported cstate change: term changed from 8 to 9, leader changed from 90f4362b094e4d1681153e2008083f4e (127.20.1.65) to afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66). New cstate: current_term: 9 leader_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: UNKNOWN } } }
18:21:13.130 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:13.130 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:13.131 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=13, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=1399), Traces: [0ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [1ms] delaying RPC due to: Illegal state: connection is terminated, [16ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [18ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [19ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [35ms] refreshing cache from master, [36ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [38ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [40ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [41ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [41ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [56ms] refreshing cache from master, [56ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [59ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [60ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [62ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [62ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [96ms] refreshing cache from master, [96ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [99ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [100ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [101ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [101ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [316ms] refreshing cache from master, [316ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [319ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [320ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [322ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [322ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [856ms] refreshing cache from master, [856ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [859ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [860ms] sending RPC to server 90f4362b094e4d1681153e2008083f4e, [862ms] received response from server 90f4362b094e4d1681153e2008083f4e: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [862ms] delaying RPC due to: Network error: Failed to connect to peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397): Connection refused: /127.20.1.65:36397, [1396ms] refreshing cache from master, [1396ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1399ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1209059719(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@4e8c11b8 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@476493971 after retry RPC@826245852) -> (continuation of Deferred@1910744080 after retry RPC@1007153554) -> (continuation of Deferred@910853312 after retry RPC@1255409336) -> (continuation of Deferred@1819760563 after retry RPC@108821827) -> (continuation of Deferred@510168053 after retry RPC@2073783070), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2b10761d -> wakeup thread Time-limited test -> (continuation of Deferred@476493971 after retry RPC@826245852) -> (continuation of Deferred@1910744080 after retry RPC@1007153554) -> (continuation of Deferred@910853312 after retry RPC@1255409336) -> (continuation of Deferred@1819760563 after retry RPC@108821827) -> (continuation of Deferred@510168053 after retry RPC@2073783070))) after lookup
18:21:13.131 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:13.132 [DEBUG - kudu-nio-4] (Connection.java:757) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] Successfully connected to peer
18:21:13.158 [DEBUG - kudu-nio-4] (Negotiator.java:918) Authenticated connection [id: 0xa794b75d, L:/127.0.0.1:32988 - R:/127.20.1.66:40881] using TOKEN/null
18:21:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.164770 22694 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 9 FOLLOWER]: Refusing update from remote peer afe5bf8ec6e44b15bad84f94f3ee7158: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 34. (index mismatch)
18:21:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:13.165392 22423 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 -> Peer 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397): Couldn't send request to peer 90f4362b094e4d1681153e2008083f4e. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:13.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.165877 22780 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
18:21:13.196 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.1.65:36397
18:21:13.196 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

18:21:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.197594 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/logs
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.65:36397
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.65
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46555
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.65
18:21:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:13.570137 22804 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:13.570488 22804 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:13.570567 22804 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:13.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:13.579241 22804 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:13.579420 22804 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.65
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.590682 22804 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/logs
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.65:36397
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.65
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46555
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.65
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:21:13.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:21:13.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.592516 22804 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:13.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.594417 22804 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:21:13.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:13.607348 22809 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:13.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:13.607410 22810 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:13.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:13.607625 22812 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.609002 22804 server_base.cc:1061] running on GCE node
18:21:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.609733 22804 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:13.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.610783 22804 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:13.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.612138 22804 hybrid_clock.cc:648] HybridClock initialized: now 1786472473612029 us; error 48 us; skew 500 ppm
18:21:13.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.612607 22804 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:21:13.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.615248 22804 webserver.cc:533] Webserver started at http://127.20.1.65:46555/ using document root <none> and password file <none>
18:21:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.616107 22804 fs_manager.cc:362] Metadata directory not provided
18:21:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.616266 22804 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.622634 22804 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.002s
18:21:13.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.625753 22819 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:13.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.627225 22804 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.627417 22804 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90f4362b094e4d1681153e2008083f4e"
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.628265 22804 fs_report.cc:389] FS layout report
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/wal
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/data
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:13.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:13.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:13.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:13.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.655282 22804 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:13.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.656327 22804 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:13.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.656695 22804 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:13.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.657781 22804 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:13.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.660763 22826 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:21:13.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.670212 22804 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:13.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.670406 22804 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.010s	user 0.001s	sys 0.000s
18:21:13.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.670586 22804 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:13.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.674639 22804 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.674839 22804 ts_tablet_manager.cc:595] Time spent register tablets: real 0.004s	user 0.001s	sys 0.000s
18:21:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.675225 22826 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Bootstrap starting.
18:21:13.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.744752 22826 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Log is configured to *not* fsync() on all Append() calls
18:21:13.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.792459 22804 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.65:36397
18:21:13.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.792752 22945 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.65:36397 every 8 connection(s)
18:21:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.794476 22804 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-0/data/info.pb
18:21:13.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.802814 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 22804
18:21:13.803 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:13.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.809360 22826 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:13.832 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:13.833 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.1.66:40881
18:21:13.833 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.834175 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 22406
18:21:13.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.834622 22826 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Bootstrap complete.
18:21:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.836722 22826 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Time spent bootstrapping tablet: real 0.162s	user 0.075s	sys 0.031s
18:21:13.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.847189 22826 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:13.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.851651 22826 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: 90f4362b094e4d1681153e2008083f4e, State: Initialized, Role: FOLLOWER
18:21:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.852723 22826 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:13.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.859402 22826 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e: Time spent starting tablet: real 0.023s	user 0.008s	sys 0.007s
18:21:13.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.862852 22946 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:21:13.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.863305 22946 heartbeater.cc:461] Registering TS with master...
18:21:13.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.864439 22946 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:21:13.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.869029 22947 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:21:13.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.869184 22947 heartbeater.cc:461] Registering TS with master...
18:21:13.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.869789 22947 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:21:13.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.870332 22546 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:13.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.870785 20699 ts_manager.cc:194] Re-registered known tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.871207 20533 ts_manager.cc:194] Re-registered known tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.872675 20699 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.65:36981
18:21:13.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.875317 22950 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:21:13.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.875517 22950 heartbeater.cc:461] Registering TS with master...
18:21:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.876005 22946 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:21:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.876030 22950 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:21:13.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.877718 20613 ts_manager.cc:194] Re-registered known tserver with Master: 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:13.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:13.980533 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 22406
18:21:14.006 [DEBUG - kudu-nio-4] (Connection.java:239) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] handling channelInactive
18:21:14.006 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:14.006 [DEBUG - kudu-nio-4] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state READY due to: connection closed
18:21:14.007 [DEBUG - Time-limited test] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:14.008 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.008 [DEBUG - kudu-nio-5] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.008 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.030 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:14.030 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:14.030 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [1ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1033429217(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1d7bafd7 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e5d5f25 -> wakeup thread Time-limited test)) after lookup
18:21:14.031 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:14.031 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.032 [DEBUG - kudu-nio-6] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.032 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.050 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:14.050 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:14.051 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [1ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [23ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [25ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [25ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1033429217(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1d7bafd7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1820763538 after retry RPC@408320848), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e5d5f25 -> wakeup thread Time-limited test -> (continuation of Deferred@1820763538 after retry RPC@408320848))) after lookup
18:21:14.051 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:14.052 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.052 [DEBUG - kudu-nio-7] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.052 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.090 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:14.090 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:14.091 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [1ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [23ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [25ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [25ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [44ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [45ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [45ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1033429217(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1d7bafd7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1820763538 after retry RPC@408320848) -> (continuation of Deferred@285212425 after retry RPC@1004047037), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e5d5f25 -> wakeup thread Time-limited test -> (continuation of Deferred@1820763538 after retry RPC@408320848) -> (continuation of Deferred@285212425 after retry RPC@1004047037))) after lookup
18:21:14.091 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:14.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.092 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.130 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:14.130 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:14.131 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=124), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [1ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [23ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [25ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [25ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [44ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [45ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [45ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [84ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [85ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [85ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [123ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1033429217(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1d7bafd7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1820763538 after retry RPC@408320848) -> (continuation of Deferred@285212425 after retry RPC@1004047037) -> (continuation of Deferred@1082203520 after retry RPC@1064565327), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e5d5f25 -> wakeup thread Time-limited test -> (continuation of Deferred@1820763538 after retry RPC@408320848) -> (continuation of Deferred@285212425 after retry RPC@1004047037) -> (continuation of Deferred@1082203520 after retry RPC@1064565327))) after lookup
18:21:14.132 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:14.133 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.133 [DEBUG - kudu-nio-1] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.133 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.190 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:14.190 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:14.191 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=183), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [1ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [23ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [25ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [25ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [44ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [45ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [45ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [84ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [85ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [85ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [123ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [124ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [126ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [126ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [180ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [183ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1033429217(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1d7bafd7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1820763538 after retry RPC@408320848) -> (continuation of Deferred@285212425 after retry RPC@1004047037) -> (continuation of Deferred@1082203520 after retry RPC@1064565327) -> (continuation of Deferred@669458338 after retry RPC@559269055), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e5d5f25 -> wakeup thread Time-limited test -> (continuation of Deferred@1820763538 after retry RPC@408320848) -> (continuation of Deferred@285212425 after retry RPC@1004047037) -> (continuation of Deferred@1082203520 after retry RPC@1064565327) -> (continuation of Deferred@669458338 after retry RPC@559269055))) after lookup
18:21:14.191 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:14.192 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.192 [DEBUG - kudu-nio-2] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state CONNECTING due to: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:14.192 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881
18:21:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.074506 22966 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 8 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:21:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.074844 22966 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:15.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.076889 22966 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:15.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.078791 22820 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:15.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.080634 22820 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881): Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:15.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.084502 22694 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "90f4362b094e4d1681153e2008083f4e" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "b5a72ae51147432e815137a6da2c75ff" is_pre_election: true
18:21:15.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.085445 22822 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e; no voters: afe5bf8ec6e44b15bad84f94f3ee7158, b5a72ae51147432e815137a6da2c75ff
18:21:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.085985 22966 raft_consensus.cc:2749] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 8 FOLLOWER]: Leader pre-election lost for term 9. Reason: could not achieve majority
18:21:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.276540 22973 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 9 FOLLOWER]: Starting pre-election (detected failure of leader afe5bf8ec6e44b15bad84f94f3ee7158)
18:21:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.276764 22973 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.278321 22973 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.279747 22621 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:15.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.281925 22621 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881): Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.287480 22899 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "b5a72ae51147432e815137a6da2c75ff" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e" is_pre_election: true
18:21:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.287955 22899 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b5a72ae51147432e815137a6da2c75ff in term 8.
18:21:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.289060 22623 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, b5a72ae51147432e815137a6da2c75ff; no voters: afe5bf8ec6e44b15bad84f94f3ee7158
18:21:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.289487 22973 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 9 FOLLOWER]: Leader pre-election won for term 10
18:21:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.289625 22973 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 9 FOLLOWER]: Starting leader election (detected failure of leader afe5bf8ec6e44b15bad84f94f3ee7158)
18:21:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.289736 22973 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 9 FOLLOWER]: Advancing to term 10
18:21:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.291839 22973 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.292570 22973 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 10 election: Requested vote from peers afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.293150 22899 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "b5a72ae51147432e815137a6da2c75ff" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e"
18:21:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.293357 22899 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 8 FOLLOWER]: Advancing to term 10
18:21:15.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.293885 22621 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881): Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111)
18:21:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.297226 22899 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b5a72ae51147432e815137a6da2c75ff in term 10.
18:21:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.297725 22623 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, b5a72ae51147432e815137a6da2c75ff; no voters: afe5bf8ec6e44b15bad84f94f3ee7158
18:21:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.298084 22973 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 10 FOLLOWER]: Leader election won for term 10
18:21:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.298688 22973 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 10 LEADER]: Becoming Leader. State: Replica: b5a72ae51147432e815137a6da2c75ff, State: Running, Role: LEADER
18:21:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.299258 22973 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:15.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.304237 20699 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff reported cstate change: term changed from 9 to 10, leader changed from afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66) to b5a72ae51147432e815137a6da2c75ff (127.20.1.67). New cstate: current_term: 10 leader_uuid: "b5a72ae51147432e815137a6da2c75ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: UNKNOWN } } }
18:21:15.490 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:15.490 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:15.490 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1483), Traces: [0ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [1ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [1ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [19ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [22ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [23ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [25ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [25ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [43ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [44ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [45ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [45ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [80ms] refreshing cache from master, [80ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [83ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [84ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [85ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [85ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [123ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [124ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [126ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [126ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [180ms] refreshing cache from master, [180ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [183ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [184ms] sending RPC to server afe5bf8ec6e44b15bad84f94f3ee7158, [185ms] received response from server afe5bf8ec6e44b15bad84f94f3ee7158: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [185ms] delaying RPC due to: Network error: Failed to connect to peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881): Connection refused: /127.20.1.66:40881, [1479ms] refreshing cache from master, [1480ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1482ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@1033429217(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@1d7bafd7 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1820763538 after retry RPC@408320848) -> (continuation of Deferred@285212425 after retry RPC@1004047037) -> (continuation of Deferred@1082203520 after retry RPC@1064565327) -> (continuation of Deferred@669458338 after retry RPC@559269055) -> (continuation of Deferred@401202831 after retry RPC@404127985), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@1e5d5f25 -> wakeup thread Time-limited test -> (continuation of Deferred@1820763538 after retry RPC@408320848) -> (continuation of Deferred@285212425 after retry RPC@1004047037) -> (continuation of Deferred@1082203520 after retry RPC@1064565327) -> (continuation of Deferred@669458338 after retry RPC@559269055) -> (continuation of Deferred@401202831 after retry RPC@404127985))) after lookup
18:21:15.491 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:15.492 [DEBUG - kudu-nio-3] (Connection.java:757) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] Successfully connected to peer
18:21:15.506 [DEBUG - kudu-nio-3] (Negotiator.java:918) Authenticated connection [id: 0x36e82ffd, L:/127.0.0.1:54758 - R:/127.20.1.67:41125] using TOKEN/null
18:21:15.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.512492 22899 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 10 FOLLOWER]: Refusing update from remote peer b5a72ae51147432e815137a6da2c75ff: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 38. (index mismatch)
18:21:15.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.512705 22621 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff -> Peer afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881): Couldn't send request to peer afe5bf8ec6e44b15bad84f94f3ee7158. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.66:40881: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:15.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.513522 22979 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [LEADER]: Connected to new peer: Peer: permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
18:21:15.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.530447 22992 mvcc.cc:204] Tried to move back new op lower bound from 7317391259688996864 to 7317391258830893056. Current Snapshot: MvccSnapshot[applied={T|T < 7317391259688996864}]
18:21:15.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.531504 22994 mvcc.cc:204] Tried to move back new op lower bound from 7317391259688996864 to 7317391258830893056. Current Snapshot: MvccSnapshot[applied={T|T < 7317391259688996864}]
18:21:15.551 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.1.66:40881
18:21:15.552 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.552803 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/logs
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.66:40881
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.66
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40147
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.66
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:15.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.922941 23002 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:15.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.923270 23002 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:15.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.923348 23002 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.932459 23002 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.932629 23002 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.66
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.943210 23002 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/logs
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.66:40881
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.66
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40147
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.66
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:21:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:21:15.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.945111 23002 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:15.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.946906 23002 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:21:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.957219 23009 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.957785 23012 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:15.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:15.957163 23010 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:15.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.958323 23002 server_base.cc:1061] running on GCE node
18:21:15.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.958981 23002 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:15.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.960037 23002 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:15.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.961270 23002 hybrid_clock.cc:648] HybridClock initialized: now 1786472475961219 us; error 45 us; skew 500 ppm
18:21:15.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.961617 23002 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:21:15.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.964030 23002 webserver.cc:533] Webserver started at http://127.20.1.66:40147/ using document root <none> and password file <none>
18:21:15.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.964859 23002 fs_manager.cc:362] Metadata directory not provided
18:21:15.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.965005 23002 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:15.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.970656 23002 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.003s	sys 0.003s
18:21:15.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.973731 23019 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:15.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.975258 23002 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.975391 23002 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afe5bf8ec6e44b15bad84f94f3ee7158"
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:49 on dist-test-slave-6k22"
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:15.976205 23002 fs_report.cc:389] FS layout report
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/wal
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/data
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.000326 23002 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:16.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.001256 23002 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:16.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.001631 23002 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:16.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.002687 23002 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:16.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.004987 23027 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:21:16.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.010852 23002 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:16.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.010974 23002 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
18:21:16.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.011119 23002 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:16.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.014405 23002 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:16.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.014530 23002 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.001s	sys 0.000s
18:21:16.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.014971 23027 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Bootstrap starting.
18:21:16.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.045153 23027 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Log is configured to *not* fsync() on all Append() calls
18:21:16.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.061057 23002 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.66:40881
18:21:16.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.061097 23167 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.66:40881 every 8 connection(s)
18:21:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.062548 23002 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-1/data/info.pb
18:21:16.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.064786 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 23002
18:21:16.065 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:16.087 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:16.089 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.20.1.67:41125
18:21:16.089 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:16.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.088492 23169 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:21:16.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.088886 23169 heartbeater.cc:461] Registering TS with master...
18:21:16.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.089699 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 22603
18:21:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.090106 23169 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:21:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.093833 20533 ts_manager.cc:194] Re-registered known tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:16.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.102012 23168 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:21:16.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.102156 23168 heartbeater.cc:461] Registering TS with master...
18:21:16.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.102807 23168 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:21:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.104790 20698 ts_manager.cc:194] Re-registered known tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.106580 20698 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.66:56503
18:21:16.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.116569 23027 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:16.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.118757 23027 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Bootstrap complete.
18:21:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.120863 23027 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Time spent bootstrapping tablet: real 0.106s	user 0.052s	sys 0.026s
18:21:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.127105 23027 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.128146 23027 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: afe5bf8ec6e44b15bad84f94f3ee7158, State: Initialized, Role: FOLLOWER
18:21:16.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.129002 23027 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:16.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.140764 23168 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:21:16.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.141804 23027 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158: Time spent starting tablet: real 0.021s	user 0.018s	sys 0.002s
18:21:16.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.146379 23106 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 9 FOLLOWER]: Advancing to term 10
18:21:16.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.151841 23171 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:21:16.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.152009 23171 heartbeater.cc:461] Registering TS with master...
18:21:16.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.152365 23171 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:21:16.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.160821 22743 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.169163 20612 ts_manager.cc:194] Re-registered known tserver with Master: afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:16.276968 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 22603
18:21:16.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:16.304 [DEBUG - kudu-nio-3] (Connection.java:239) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] handling channelInactive
18:21:16.304 [DEBUG - kudu-nio-3] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state READY due to: connection closed
18:21:16.307 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:16.308 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.308 [DEBUG - kudu-nio-4] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.308 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.330 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:16.330 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:16.330 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=3, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=26), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [4ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [4ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [25ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@2067314176(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5785a09e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63e195ff -> wakeup thread Time-limited test)) after lookup
18:21:16.331 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:16.332 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.332 [DEBUG - kudu-nio-5] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.332 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.350 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:16.351 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:16.351 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=5, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=47), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [4ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [4ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [25ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [26ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [28ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [28ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@2067314176(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5785a09e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1708781538 after retry RPC@175689931), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63e195ff -> wakeup thread Time-limited test -> (continuation of Deferred@1708781538 after retry RPC@175689931))) after lookup
18:21:16.351 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:16.352 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.352 [DEBUG - kudu-nio-6] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.353 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.390 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:16.391 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:16.391 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=7, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=87), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [4ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [4ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [25ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [26ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [28ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [28ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [47ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [49ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [49ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [86ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@2067314176(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5785a09e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1708781538 after retry RPC@175689931) -> (continuation of Deferred@798613983 after retry RPC@1672188527), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63e195ff -> wakeup thread Time-limited test -> (continuation of Deferred@1708781538 after retry RPC@175689931) -> (continuation of Deferred@798613983 after retry RPC@1672188527))) after lookup
18:21:16.392 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:16.393 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.393 [DEBUG - kudu-nio-7] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.393 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.570 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:16.570 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:16.570 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=9, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=266), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [4ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [4ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [25ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [26ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [28ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [28ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [47ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [49ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [49ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [86ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [87ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [89ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [89ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [263ms] refreshing cache from master, [263ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [266ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@2067314176(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5785a09e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1708781538 after retry RPC@175689931) -> (continuation of Deferred@798613983 after retry RPC@1672188527) -> (continuation of Deferred@403119444 after retry RPC@662752465), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63e195ff -> wakeup thread Time-limited test -> (continuation of Deferred@1708781538 after retry RPC@175689931) -> (continuation of Deferred@798613983 after retry RPC@1672188527) -> (continuation of Deferred@403119444 after retry RPC@662752465))) after lookup
18:21:16.571 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:16.572 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.572 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:16.572 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.496982 23196 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 10 FOLLOWER]: Starting pre-election (detected failure of leader b5a72ae51147432e815137a6da2c75ff)
18:21:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.497234 23196 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:17.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.498096 23196 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66:40881)
18:21:17.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:17.500356 22822 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.505100 23106 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "90f4362b094e4d1681153e2008083f4e" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" is_pre_election: true
18:21:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.506182 22820 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e; no voters: afe5bf8ec6e44b15bad84f94f3ee7158, b5a72ae51147432e815137a6da2c75ff
18:21:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.506464 23196 raft_consensus.cc:2749] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 10 FOLLOWER]: Leader pre-election lost for term 11. Reason: could not achieve majority
18:21:17.549 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:17.550 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881),b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)[L]]}]
18:21:17.550 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=11, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=1246), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [4ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [4ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [25ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [26ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [28ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [28ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [47ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [49ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [49ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [86ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [87ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [89ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [89ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [263ms] refreshing cache from master, [263ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [266ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [267ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [268ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [268ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [1243ms] refreshing cache from master, [1243ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1245ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@2067314176(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5785a09e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1708781538 after retry RPC@175689931) -> (continuation of Deferred@798613983 after retry RPC@1672188527) -> (continuation of Deferred@403119444 after retry RPC@662752465) -> (continuation of Deferred@1548645168 after retry RPC@2097793580), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63e195ff -> wakeup thread Time-limited test -> (continuation of Deferred@1708781538 after retry RPC@175689931) -> (continuation of Deferred@798613983 after retry RPC@1672188527) -> (continuation of Deferred@403119444 after retry RPC@662752465) -> (continuation of Deferred@1548645168 after retry RPC@2097793580))) after lookup
18:21:17.551 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] connecting to peer
18:21:17.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:17.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] cleaning up while in state CONNECTING due to: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:17.552 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125) for tablet c309490fd3a441c9b1a125d394f3629e: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125
18:21:17.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.653821 23201 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader b5a72ae51147432e815137a6da2c75ff)
18:21:17.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.654074 23201 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:17.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.655491 23201 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:17.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:17.657214 23022 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:17.659060 23022 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.663144 22899 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e" is_pre_election: true
18:21:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.663408 22899 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 10 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate afe5bf8ec6e44b15bad84f94f3ee7158 in term 10.
18:21:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.664093 23022 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, afe5bf8ec6e44b15bad84f94f3ee7158; no voters: b5a72ae51147432e815137a6da2c75ff
18:21:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.664495 23201 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 10 FOLLOWER]: Leader pre-election won for term 11
18:21:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.664645 23201 raft_consensus.cc:493] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 10 FOLLOWER]: Starting leader election (detected failure of leader b5a72ae51147432e815137a6da2c75ff)
18:21:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.664767 23201 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 10 FOLLOWER]: Advancing to term 11
18:21:17.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.666858 23201 raft_consensus.cc:515] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:17.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.667462 23201 leader_election.cc:290] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 11 election: Requested vote from peers b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125), 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397)
18:21:17.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.668332 22899 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "c309490fd3a441c9b1a125d394f3629e" candidate_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "90f4362b094e4d1681153e2008083f4e"
18:21:17.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.668591 22899 raft_consensus.cc:3060] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 10 FOLLOWER]: Advancing to term 11
18:21:17.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:17.668944 23022 leader_election.cc:336] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111)
18:21:17.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.670644 22899 raft_consensus.cc:2468] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate afe5bf8ec6e44b15bad84f94f3ee7158 in term 11.
18:21:17.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.671060 23022 leader_election.cc:304] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 90f4362b094e4d1681153e2008083f4e, afe5bf8ec6e44b15bad84f94f3ee7158; no voters: b5a72ae51147432e815137a6da2c75ff
18:21:17.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.671370 23201 raft_consensus.cc:2804] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 11 FOLLOWER]: Leader election won for term 11
18:21:17.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.671815 23201 raft_consensus.cc:697] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [term 11 LEADER]: Becoming Leader. State: Replica: afe5bf8ec6e44b15bad84f94f3ee7158, State: Running, Role: LEADER
18:21:17.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.672408 23201 consensus_queue.cc:237] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:17.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:17.676296 20698 catalog_manager.cc:5719] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 reported cstate change: term changed from 10 to 11, leader changed from b5a72ae51147432e815137a6da2c75ff (127.20.1.67) to afe5bf8ec6e44b15bad84f94f3ee7158 (127.20.1.66). New cstate: current_term: 11 leader_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } health_report { overall_health: UNKNOWN } } }
18:21:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:18.211863 22899 raft_consensus.cc:1275] T c309490fd3a441c9b1a125d394f3629e P 90f4362b094e4d1681153e2008083f4e [term 11 FOLLOWER]: Refusing update from remote peer afe5bf8ec6e44b15bad84f94f3ee7158: Log matching property violated. Preceding OpId in replica: term: 10 index: 40. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
18:21:18.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:18.212695 23224 consensus_queue.cc:1048] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 [LEADER]: Connected to new peer: Peer: permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 40, Time since last communication: 0.000s
18:21:18.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:18.223263 23022 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 -> Peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Couldn't send request to peer b5a72ae51147432e815137a6da2c75ff. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:20.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:20.851480 23022 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 -> Peer b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125): Couldn't send request to peer b5a72ae51147432e815137a6da2c75ff. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.67:41125: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
18:21:21.190 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet c309490fd3a441c9b1a125d394f3629e for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330' with partition [<start>, <end>)
18:21:21.190 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=c309490fd3a441c9b1a125d394f3629e@[90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397),afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)[L],b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)]}]
18:21:21.190 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=c309490fd3a441c9b1a125d394f3629e, attempt=13, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=4886), Traces: [0ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [0ms] delaying RPC due to: Illegal state: connection is terminated, [3ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [4ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [4ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [23ms] refreshing cache from master, [23ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [25ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [26ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [28ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [28ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [43ms] refreshing cache from master, [43ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [46ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [47ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [49ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [49ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [83ms] refreshing cache from master, [83ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [86ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [87ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [89ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [89ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [263ms] refreshing cache from master, [263ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [266ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [267ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [268ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [268ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [1243ms] refreshing cache from master, [1243ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [1245ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, [1247ms] sending RPC to server b5a72ae51147432e815137a6da2c75ff, [1248ms] received response from server b5a72ae51147432e815137a6da2c75ff: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [1248ms] delaying RPC due to: Network error: Failed to connect to peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125): Connection refused: /127.20.1.67:41125, [4882ms] refreshing cache from master, [4883ms] Sub RPC GetTableLocations: sending RPC to server master-127.20.1.124:35869, [4885ms] Sub RPC GetTableLocations: received response from server master-127.20.1.124:35869: OK, deferred=Deferred@2067314176(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$234/0x00007f29e836b560@5785a09e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1708781538 after retry RPC@175689931) -> (continuation of Deferred@798613983 after retry RPC@1672188527) -> (continuation of Deferred@403119444 after retry RPC@662752465) -> (continuation of Deferred@1548645168 after retry RPC@2097793580) -> (continuation of Deferred@371545778 after retry RPC@457559939), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@63e195ff -> wakeup thread Time-limited test -> (continuation of Deferred@1708781538 after retry RPC@175689931) -> (continuation of Deferred@798613983 after retry RPC@1672188527) -> (continuation of Deferred@403119444 after retry RPC@662752465) -> (continuation of Deferred@1548645168 after retry RPC@2097793580) -> (continuation of Deferred@371545778 after retry RPC@457559939))) after lookup
18:21:21.191 [DEBUG - kudu-nio-1] (Connection.java:748) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] connecting to peer
18:21:21.192 [DEBUG - kudu-nio-2] (Connection.java:757) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] Successfully connected to peer
18:21:21.208 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x7c1adccb, L:/127.0.0.1:39418 - R:/127.20.1.66:40881] using TOKEN/null
18:21:21.236 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.20.1.67:41125
18:21:21.236 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.237200 20485 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/logs
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.67:41125
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.67
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46629
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.67
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:21.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:21.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:21:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:21.588718 23253 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
18:21:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:21.589032 23253 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:21:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:21.589107 23253 flags.cc:432] Enabled unsafe flag: --never_fsync=true
18:21:21.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:21.597673 23253 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
18:21:21.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:21.597846 23253 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.1.67
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.609051 23253 tablet_server_runner.cc:78] Tablet server non-default flags:
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/logs
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.1.84:42677
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.1.67:41125
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.1.67
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46629
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.1.126:36795,127.20.1.125:35653,127.20.1.124:35869
18:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.1.67
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cfdbd367ce7fd9747cb6f1860fc64263d9f9f79a
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 11 Aug 2026 17:57:42 UTC on e7f111948823
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 17631
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
18:21:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.610790 23253 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:21:21.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.612706 23253 file_cache.cc:504] Constructed file cache file cache with capacity 419430
18:21:21.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:21.623896 23259 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:21.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:21.623883 23260 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:21.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:21.625586 23263 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:21:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.625973 23253 server_base.cc:1061] running on GCE node
18:21:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.626650 23253 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:21:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.627797 23253 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:21:21.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.629026 23253 hybrid_clock.cc:648] HybridClock initialized: now 1786472481628970 us; error 50 us; skew 500 ppm
18:21:21.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.629316 23253 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:21:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.631707 23253 webserver.cc:533] Webserver started at http://127.20.1.67:46629/ using document root <none> and password file <none>
18:21:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.632460 23253 fs_manager.cc:362] Metadata directory not provided
18:21:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.632558 23253 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:21:21.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.638339 23253 fs_manager.cc:714] Time spent opening directory manager: real 0.004s	user 0.005s	sys 0.000s
18:21:21.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.641408 23269 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
18:21:21.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.642807 23253 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.000s
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.642949 23253 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data,/tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5a72ae51147432e815137a6da2c75ff"
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 18:20:50 on dist-test-slave-6k22"
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.643970 23253 fs_report.cc:389] FS layout report
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/wal
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/data
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:21:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:21:21.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.666263 23253 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
18:21:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.667295 23253 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:21:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.667703 23253 kserver.cc:163] Server-wide thread pool size limit: 3276
18:21:21.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.668816 23253 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
18:21:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.671028 23277 ts_tablet_manager.cc:548] Loading tablet metadata (0/1 complete)
18:21:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.677026 23253 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
18:21:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.677170 23253 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.006s	user 0.001s	sys 0.000s
18:21:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.677314 23253 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
18:21:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.680590 23253 ts_tablet_manager.cc:616] Registered 1 tablets
18:21:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.680724 23253 ts_tablet_manager.cc:595] Time spent register tablets: real 0.003s	user 0.003s	sys 0.000s
18:21:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.681209 23277 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap starting.
18:21:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.720296 23277 log.cc:826] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Log is configured to *not* fsync() on all Append() calls
18:21:21.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.757619 23253 rpc_server.cc:307] RPC server started. Bound to: 127.20.1.67:41125
18:21:21.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.757877 23400 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.1.67:41125 every 8 connection(s)
18:21:21.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.759428 23253 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskiKkpA8/test-tmp/mini-kudu-cluster4359800343100928179/ts-2/data/info.pb
18:21:21.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.760031 20485 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu as pid 23253
18:21:21.760 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
18:21:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.779062 23277 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap replayed 1/1 log segments. Stats: ops{read=40 overwritten=0 applied=40 ignored=0} inserts{seen=30 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:21:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.780128 23277 tablet_bootstrap.cc:492] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Bootstrap complete.
18:21:21.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.781988 23277 ts_tablet_manager.cc:1403] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Time spent bootstrapping tablet: real 0.101s	user 0.071s	sys 0.027s
18:21:21.787 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:21.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.789815 23277 raft_consensus.cc:359] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 10 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:21.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.790967 23277 raft_consensus.cc:740] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [term 10 FOLLOWER]: Becoming Follower/Learner. State: Replica: b5a72ae51147432e815137a6da2c75ff, State: Initialized, Role: FOLLOWER
18:21:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.792119 23277 consensus_queue.cc:260] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5a72ae51147432e815137a6da2c75ff" member_type: VOTER last_known_addr { host: "127.20.1.67" port: 41125 } } peers { permanent_uuid: "afe5bf8ec6e44b15bad84f94f3ee7158" member_type: VOTER last_known_addr { host: "127.20.1.66" port: 40881 } } peers { permanent_uuid: "90f4362b094e4d1681153e2008083f4e" member_type: VOTER last_known_addr { host: "127.20.1.65" port: 36397 } }
18:21:21.805 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1786472451330, tablet=c309490fd3a441c9b1a125d394f3629e, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
18:21:21.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.809959 23277 ts_tablet_manager.cc:1434] T c309490fd3a441c9b1a125d394f3629e P b5a72ae51147432e815137a6da2c75ff: Time spent starting tablet: real 0.028s	user 0.017s	sys 0.002s
18:21:21.814 [DEBUG - main] (Connection.java:575) [peer master-127.20.1.125:35653(127.20.1.125:35653)] disconnecting while in state READY
18:21:21.816 [DEBUG - main] (Connection.java:575) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] disconnecting while in state READY
18:21:21.816 [DEBUG - main] (Connection.java:575) [peer 90f4362b094e4d1681153e2008083f4e(127.20.1.65:36397)] disconnecting while in state TERMINATED
18:21:21.816 [DEBUG - main] (Connection.java:575) [peer b5a72ae51147432e815137a6da2c75ff(127.20.1.67:41125)] disconnecting while in state TERMINATED
18:21:21.817 [DEBUG - main] (Connection.java:575) [peer master-127.20.1.126:36795(127.20.1.126:36795)] disconnecting while in state READY
18:21:21.818 [DEBUG - main] (Connection.java:575) [peer master-127.20.1.124:35869(127.20.1.124:35869)] disconnecting while in state READY
18:21:21.818 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.1.126:36795(127.20.1.126:36795)] handling channelInactive
18:21:21.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.1.126:36795(127.20.1.126:36795)] cleaning up while in state READY due to: connection closed
18:21:21.819 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.1.124:35869(127.20.1.124:35869)] handling channelInactive
18:21:21.820 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.20.1.125:35653(127.20.1.125:35653)] handling channelInactive
18:21:21.820 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.20.1.125:35653(127.20.1.125:35653)] cleaning up while in state READY due to: connection closed
18:21:21.821 [DEBUG - kudu-nio-2] (Connection.java:239) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] handling channelInactive
18:21:21.821 [DEBUG - kudu-nio-2] (Connection.java:709) [peer afe5bf8ec6e44b15bad84f94f3ee7158(127.20.1.66:40881)] cleaning up while in state READY due to: connection closed
18:21:21.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.1.124:35869(127.20.1.124:35869)] cleaning up while in state READY due to: connection closed
18:21:21.822 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
18:21:21.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.829695 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 22804
18:21:21.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.833752 23402 heartbeater.cc:344] Connected to a master server at 127.20.1.124:35869
18:21:21.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.834357 23402 heartbeater.cc:461] Registering TS with master...
18:21:21.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.836040 23402 heartbeater.cc:507] Master 127.20.1.124:35869 requested a full tablet report, sending...
18:21:21.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.843806 20698 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:21:21.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.846005 20698 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.20.1.67:40375
18:21:21.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.850519 23402 heartbeater.cc:499] Master 127.20.1.124:35869 was elected leader, sending a full tablet report...
18:21:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.872929 23403 heartbeater.cc:344] Connected to a master server at 127.20.1.126:36795
18:21:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.873121 23403 heartbeater.cc:461] Registering TS with master...
18:21:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.873538 23403 heartbeater.cc:507] Master 127.20.1.126:36795 requested a full tablet report, sending...
18:21:21.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.875243 23408 heartbeater.cc:344] Connected to a master server at 127.20.1.125:35653
18:21:21.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.875478 23408 heartbeater.cc:461] Registering TS with master...
18:21:21.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.875619 20533 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:21:21.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.876008 23408 heartbeater.cc:507] Master 127.20.1.125:35653 requested a full tablet report, sending...
18:21:21.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.877651 20613 ts_manager.cc:194] Re-registered known tserver with Master: b5a72ae51147432e815137a6da2c75ff (127.20.1.67:41125)
18:21:21.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.877722 22941 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:21.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:21.979753 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 22804
18:21:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.009944 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 23002
18:21:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.019374 23162 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:22.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 18:21:22.135012 23022 consensus_peers.cc:597] T c309490fd3a441c9b1a125d394f3629e P afe5bf8ec6e44b15bad84f94f3ee7158 -> Peer 90f4362b094e4d1681153e2008083f4e (127.20.1.65:36397): Couldn't send request to peer 90f4362b094e4d1681153e2008083f4e. Status: Network error: Client connection negotiation failed: client connection to 127.20.1.65:36397: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
18:21:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.136590 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 23002
18:21:22.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.163173 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 23253
18:21:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.172236 23396 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:22.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.268352 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 23253
18:21:22.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.292565 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 20498
18:21:22.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.293944 20568 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:22.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.416596 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 20498
18:21:22.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.445922 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 20578
18:21:22.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.447180 20647 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:22.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.565953 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 20578
18:21:22.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.596179 20485 external_mini_cluster.cc:1705] Attempting to check leaks for /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu pid 20654
18:21:22.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.597489 20735 generic_service.cc:196] Checking for leaks (request via RPC)
18:21:22.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 18:21:22.718408 20485 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskiKkpA8/build/asan/bin/kudu with pid 20654
18:21:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T18:21:22Z chronyd exiting

Time: 37.908

OK (1 test)

