JUnit version 4.13.2
.20:03:37.656 [DEBUG - main] (RandomUtils.java:49) Using random seed: 1779134617649
20:03:37.718 [INFO - main] (RetryRule.java:138) Creating RetryStatement with result reporter and retry count of 0 (no retries)
20:03:37.790 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:03:37.804 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskswF7jE/test-tmp
20:03:37.810 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
20:03:37.812 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskswF7jE/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
20:03:38.004 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

20:03:38.018 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
20:03:38.020 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

20:03:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.022393  4785 mini_oidc.cc:150] Starting JWKS server
20:03:38.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.024674  4785 webserver.cc:533] Webserver started at http://127.0.0.1:40719/ using document root <none> and password file <none>
20:03:38.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.024878  4785 mini_oidc.cc:186] Starting OIDC Discovery server
20:03:38.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.025161  4785 webserver.cc:533] Webserver started at http://127.0.0.1:43741/ using document root <none> and password file <none>
20:03:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T20:03:38Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:03:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T20:03:38Z Disabled control of system clock
20:03:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.042481  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/wal
20:03:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/data
20:03:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:38.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:38.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/logs
20:03:38.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/data/info.pb
20:03:38.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:38.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:38.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:38.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:38.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:38.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:38.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:38.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:38.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:38.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:38.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:38.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:38.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.126:44523
20:03:38.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.126
20:03:38.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:38.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:38.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:38.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:38.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:38.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40719/jwks.json with env {}
20:03:38.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.148005  4796 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:38.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.148272  4796 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:38.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.148330  4796 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:38.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.151993  4796 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:38.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.152063  4796 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:38.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.152091  4796 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:38.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.152113  4796 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:38.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.152137  4796 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:38.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.152156  4796 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:40719/jwks.json
20:03:38.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.156502  4796 master_runner.cc:387] Master server non-default flags:
20:03:38.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:38.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:38.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:38.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:38.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/data
20:03:38.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/wal
20:03:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:38.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.126:44523
20:03:38.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:38.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:38.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:38.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40719/jwks.json
20:03:38.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:38.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:38.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/data/info.pb
20:03:38.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.126
20:03:38.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:38.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:38.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4796
20:03:38.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:38.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:38.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:38.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:38.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/logs
20:03:38.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:38.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:38.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:38.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:38.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:38.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:38.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.157979  4796 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.159484  4796 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:38.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.166677  4804 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.166627  4802 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.166638  4801 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.166821  4796 server_base.cc:1061] running on GCE node
20:03:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.167448  4796 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.168509  4796 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:38.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.169664  4796 hybrid_clock.cc:648] HybridClock initialized: now 1779134618169640 us; error 40 us; skew 500 ppm
20:03:38.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.171445  4796 webserver.cc:533] Webserver started at http://127.4.172.126:40013/ using document root <none> and password file <none>
20:03:38.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.172067  4796 fs_manager.cc:362] Metadata directory not provided
20:03:38.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.172127  4796 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:38.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.172358  4796 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.174008  4796 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/data/instance:
20:03:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "991ed6bcbbeb454f870239f56067dfc8"
20:03:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.174571  4796 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/wal/instance:
20:03:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "991ed6bcbbeb454f870239f56067dfc8"
20:03:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.178176  4796 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.001s	sys 0.001s
20:03:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.180375  4810 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.181378  4796 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.000s	sys 0.002s
20:03:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.181489  4796 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/wal
20:03:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "991ed6bcbbeb454f870239f56067dfc8"
20:03:38.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.181591  4796 fs_report.cc:389] FS layout report
20:03:38.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:38.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/wal
20:03:38.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/wal
20:03:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/data/data
20:03:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:38.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:38.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:38.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:38.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:38.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.194504  4796 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:38.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.195297  4796 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:38.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.195477  4796 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.202339  4863 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.126:44523 every 8 connection(s)
20:03:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.202343  4796 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.126:44523
20:03:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.203552  4796 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/data/info.pb
20:03:38.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.207062  4864 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:38.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.209461  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 4796
20:03:38.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.209599  4785 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-0/wal/instance
20:03:38.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.211123  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:38.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:38.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/wal
20:03:38.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/data
20:03:38.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:38.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:38.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:38.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:38.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/logs
20:03:38.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/data/info.pb
20:03:38.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:38.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:38.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.125:33165
20:03:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.125
20:03:38.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40719/jwks.json with env {}
20:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.211233  4864 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } has no permanent_uuid. Determining permanent_uuid...
20:03:38.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.223541  4864 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } has no permanent_uuid. Determining permanent_uuid...
20:03:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.224260  4811 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.125:33165: connect: Connection refused (error 111)
20:03:38.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.225062  4864 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.172.125:33165: Network error: Client connection negotiation failed: client connection to 127.4.172.125:33165: connect: Connection refused (error 111)
20:03:38.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.274296  4864 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } attempt: 1
20:03:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.275694  4864 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.172.125:33165: Network error: Client connection negotiation failed: client connection to 127.4.172.125:33165: connect: Connection refused (error 111)
20:03:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.313547  4866 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.313735  4866 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:38.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.313776  4866 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.317052  4866 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.317106  4866 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.317130  4866 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.317147  4866 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.317162  4866 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.317174  4866 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:40719/jwks.json
20:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.320938  4866 master_runner.cc:387] Master server non-default flags:
20:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/data
20:03:38.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/wal
20:03:38.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:38.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:38.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:38.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:38.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.125:33165
20:03:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40719/jwks.json
20:03:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/data/info.pb
20:03:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.125
20:03:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4866
20:03:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:38.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/logs
20:03:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.321910  4866 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.322999  4866 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.328917  4874 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.328905  4875 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.329108  4877 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.330591  4866 server_base.cc:1061] running on GCE node
20:03:38.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.331146  4866 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.332063  4866 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.333227  4866 hybrid_clock.cc:648] HybridClock initialized: now 1779134618333213 us; error 23 us; skew 500 ppm
20:03:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.334925  4866 webserver.cc:533] Webserver started at http://127.4.172.125:43677/ using document root <none> and password file <none>
20:03:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.335575  4866 fs_manager.cc:362] Metadata directory not provided
20:03:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.335652  4866 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:38.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.335855  4866 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.337620  4866 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/data/instance:
20:03:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fde830cced344bd3bb684650ace094f3"
20:03:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.338034  4866 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/wal/instance:
20:03:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fde830cced344bd3bb684650ace094f3"
20:03:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.341905  4866 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.006s	sys 0.000s
20:03:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.343889  4864 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } attempt: 2
20:03:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.344718  4883 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:38.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.345332  4864 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.172.125:33165: Network error: Client connection negotiation failed: client connection to 127.4.172.125:33165: connect: Connection refused (error 111)
20:03:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.346294  4866 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.000s
20:03:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.346398  4866 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/wal
20:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fde830cced344bd3bb684650ace094f3"
20:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.346464  4866 fs_report.cc:389] FS layout report
20:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/wal
20:03:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/wal
20:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/data/data
20:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:38.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.368651  4866 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.369326  4866 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.369467  4866 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.376375  4935 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.125:33165 every 8 connection(s)
20:03:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.376379  4866 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.125:33165
20:03:38.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.377513  4866 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/data/info.pb
20:03:38.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.377862  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 4866
20:03:38.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.377959  4785 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-1/wal/instance
20:03:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.378700  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/wal
20:03:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/data
20:03:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/logs
20:03:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/data/info.pb
20:03:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.124:33635
20:03:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.124
20:03:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40719/jwks.json with env {}
20:03:38.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.389544  4937 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.392550  4937 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } has no permanent_uuid. Determining permanent_uuid...
20:03:38.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.400733  4937 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } has no permanent_uuid. Determining permanent_uuid...
20:03:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.406512  4937 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } has no permanent_uuid. Determining permanent_uuid...
20:03:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.407284  4884 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.124:33635: connect: Connection refused (error 111)
20:03:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.408238  4937 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.172.124:33635: Network error: Client connection negotiation failed: client connection to 127.4.172.124:33635: connect: Connection refused (error 111)
20:03:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.436492  4864 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } attempt: 3
20:03:38.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.440711  4864 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } has no permanent_uuid. Determining permanent_uuid...
20:03:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.441834  4864 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.172.124:33635: Network error: Client connection negotiation failed: client connection to 127.4.172.124:33635: connect: Connection refused (error 111)
20:03:38.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.457427  4937 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } attempt: 1
20:03:38.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.458689  4937 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.172.124:33635: Network error: Client connection negotiation failed: client connection to 127.4.172.124:33635: connect: Connection refused (error 111)
20:03:38.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.472990  4864 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } attempt: 1
20:03:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.474316  4864 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.172.124:33635: Network error: Client connection negotiation failed: client connection to 127.4.172.124:33635: connect: Connection refused (error 111)
20:03:38.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.484400  4936 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:38.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.484603  4936 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:38.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.484645  4936 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.487890  4936 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.487942  4936 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.487962  4936 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.487977  4936 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.487993  4936 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:38.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.488005  4936 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:40719/jwks.json
20:03:38.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.491778  4936 master_runner.cc:387] Master server non-default flags:
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/data
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/wal
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.124:33635
20:03:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40719/jwks.json
20:03:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:38.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/data/info.pb
20:03:38.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.124
20:03:38.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:38.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:38.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4936
20:03:38.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:38.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/logs
20:03:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.493062  4936 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:38.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.494413  4936 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:38.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.500492  4946 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.500640  4947 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.500747  4936 server_base.cc:1061] running on GCE node
20:03:38.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.500952  4949 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.503230  4936 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:38.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.504180  4936 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:38.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.505589  4936 hybrid_clock.cc:648] HybridClock initialized: now 1779134618505527 us; error 79 us; skew 500 ppm
20:03:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.507383  4936 webserver.cc:533] Webserver started at http://127.4.172.124:46587/ using document root <none> and password file <none>
20:03:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.508038  4936 fs_manager.cc:362] Metadata directory not provided
20:03:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.508100  4936 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.508342  4936 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:38.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.510094  4936 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/data/instance:
20:03:38.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35eb0e66fe984442b2cb13b39c6a1a71"
20:03:38.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.510633  4936 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/wal/instance:
20:03:38.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35eb0e66fe984442b2cb13b39c6a1a71"
20:03:38.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.513938  4936 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.001s	sys 0.001s
20:03:38.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.516090  4955 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:38.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.516996  4936 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.000s	sys 0.001s
20:03:38.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.517098  4936 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/wal
20:03:38.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35eb0e66fe984442b2cb13b39c6a1a71"
20:03:38.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.517203  4936 fs_report.cc:389] FS layout report
20:03:38.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:38.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/wal
20:03:38.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/wal
20:03:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/data/data
20:03:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:38.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:38.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:38.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.526861  4937 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } attempt: 2
20:03:38.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.528101  4937 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.172.124:33635: Network error: Client connection negotiation failed: client connection to 127.4.172.124:33635: connect: Connection refused (error 111)
20:03:38.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.548408  4936 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.549089  4936 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.549244  4936 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.549479  4864 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } attempt: 2
20:03:38.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.550908  4864 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.4.172.124:33635: Network error: Client connection negotiation failed: client connection to 127.4.172.124:33635: connect: Connection refused (error 111)
20:03:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.556768  5007 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.124:33635 every 8 connection(s)
20:03:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.556775  4936 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.124:33635
20:03:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.558022  4936 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/data/info.pb
20:03:38.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.560956  5008 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.564671  5008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } has no permanent_uuid. Determining permanent_uuid...
20:03:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.566213  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 4936
20:03:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.566322  4785 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/master-2/wal/instance
20:03:38.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.573966  5008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } has no permanent_uuid. Determining permanent_uuid...
20:03:38.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.578197  5008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } has no permanent_uuid. Determining permanent_uuid...
20:03:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.587172  5008 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71: Bootstrap starting.
20:03:38.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.589484  5008 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71: Neither blocks nor log segments found. Creating new log.
20:03:38.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.590376  5008 log.cc:826] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71: Log is configured to *not* fsync() on all Append() calls
20:03:38.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.592417  5008 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71: No bootstrap required, opened a new log
20:03:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.595302  5008 raft_consensus.cc:359] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } }
20:03:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.595481  5008 raft_consensus.cc:385] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.595523  5008 raft_consensus.cc:740] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 35eb0e66fe984442b2cb13b39c6a1a71, State: Initialized, Role: FOLLOWER
20:03:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.595907  5008 consensus_queue.cc:260] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } }
20:03:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.596393  5017 sys_catalog.cc:455] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } } }
20:03:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.596542  5017 sys_catalog.cc:458] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [sys.catalog]: This master's current role is: FOLLOWER
20:03:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.597121  5008 sys_catalog.cc:565] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [sys.catalog]: configured and running, proceeding with master startup.
20:03:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.599990  5017 raft_consensus.cc:493] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.600119  5017 raft_consensus.cc:515] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } }
20:03:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.601075  5017 leader_election.cc:290] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 991ed6bcbbeb454f870239f56067dfc8 (127.4.172.126:44523), fde830cced344bd3bb684650ace094f3 (127.4.172.125:33165)
20:03:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.601967  5028 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.602294  5028 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.600668  4936 master_runner.cc:187] Error getting master registration for 127.4.172.126:44523: OK, instance_id { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" instance_seqno: 1779134618200560 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.602835  4936 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:03:38.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.619282  4937 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } attempt: 3
20:03:38.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.627718  4937 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3: Bootstrap starting.
20:03:38.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.630693  4937 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3: Neither blocks nor log segments found. Creating new log.
20:03:38.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.631886  4937 log.cc:826] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3: Log is configured to *not* fsync() on all Append() calls
20:03:38.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.634284  4937 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3: No bootstrap required, opened a new log
20:03:38.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.637701  4937 raft_consensus.cc:359] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } }
20:03:38.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.637909  4937 raft_consensus.cc:385] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:38.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.637959  4937 raft_consensus.cc:740] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fde830cced344bd3bb684650ace094f3, State: Initialized, Role: FOLLOWER
20:03:38.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.638465  4937 consensus_queue.cc:260] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } }
20:03:38.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.639087  5031 sys_catalog.cc:455] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } } }
20:03:38.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.639287  5031 sys_catalog.cc:458] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [sys.catalog]: This master's current role is: FOLLOWER
20:03:38.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.639780  4937 sys_catalog.cc:565] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [sys.catalog]: configured and running, proceeding with master startup.
20:03:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.639873  4910 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fde830cced344bd3bb684650ace094f3" is_pre_election: true
20:03:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.640131  4910 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 35eb0e66fe984442b2cb13b39c6a1a71 in term 0.
20:03:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.641119  4956 leader_election.cc:304] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 35eb0e66fe984442b2cb13b39c6a1a71, fde830cced344bd3bb684650ace094f3; no voters: 
20:03:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.641434  5017 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:03:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.641525  5017 raft_consensus.cc:493] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:03:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.641566  5017 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 0 FOLLOWER]: Advancing to term 1
20:03:38.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.642719  5017 raft_consensus.cc:515] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } }
20:03:38.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.643147  5017 leader_election.cc:290] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [CANDIDATE]: Term 1 election: Requested vote from peers 991ed6bcbbeb454f870239f56067dfc8 (127.4.172.126:44523), fde830cced344bd3bb684650ace094f3 (127.4.172.125:33165)
20:03:38.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.648736  4910 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fde830cced344bd3bb684650ace094f3"
20:03:38.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.648891  4910 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [term 0 FOLLOWER]: Advancing to term 1
20:03:38.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.650187  4910 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 35eb0e66fe984442b2cb13b39c6a1a71 in term 1.
20:03:38.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.650381  4864 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } attempt: 3
20:03:38.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.650874  4956 leader_election.cc:304] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 35eb0e66fe984442b2cb13b39c6a1a71, fde830cced344bd3bb684650ace094f3; no voters: 
20:03:38.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.651417  5017 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 1 FOLLOWER]: Leader election won for term 1
20:03:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.651762  5017 raft_consensus.cc:697] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [term 1 LEADER]: Becoming Leader. State: Replica: 35eb0e66fe984442b2cb13b39c6a1a71, State: Running, Role: LEADER
20:03:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.651907  5042 catalog_manager.cc:1593] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.651976  5042 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.650068  4866 master_runner.cc:187] Error getting master registration for 127.4.172.126:44523: OK, instance_id { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" instance_seqno: 1779134618200560 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.652343  4866 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:03:38.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.652637  5017 consensus_queue.cc:237] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } }
20:03:38.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.656195  5017 sys_catalog.cc:455] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 35eb0e66fe984442b2cb13b39c6a1a71. Latest consensus state: current_term: 1 leader_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } } }
20:03:38.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.656319  5017 sys_catalog.cc:458] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [sys.catalog]: This master's current role is: LEADER
20:03:38.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.660966  4864 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8: Bootstrap starting.
20:03:38.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.662469  5046 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
20:03:38.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.663481  5046 catalog_manager.cc:1519] Initializing Kudu cluster ID...
20:03:38.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.663604  4864 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8: Neither blocks nor log segments found. Creating new log.
20:03:38.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.664674  4864 log.cc:826] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8: Log is configured to *not* fsync() on all Append() calls
20:03:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.667428  4864 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8: No bootstrap required, opened a new log
20:03:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.670660  4864 raft_consensus.cc:359] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } }
20:03:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.670836  4864 raft_consensus.cc:385] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.670881  4864 raft_consensus.cc:740] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 991ed6bcbbeb454f870239f56067dfc8, State: Initialized, Role: FOLLOWER
20:03:38.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.671995  4864 consensus_queue.cc:260] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } }
20:03:38.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.676077  4864 sys_catalog.cc:565] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [sys.catalog]: configured and running, proceeding with master startup.
20:03:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.676148  4838 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "991ed6bcbbeb454f870239f56067dfc8" is_pre_election: true
20:03:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.678755  4838 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 35eb0e66fe984442b2cb13b39c6a1a71 in term 0.
20:03:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.676301  4837 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "991ed6bcbbeb454f870239f56067dfc8"
20:03:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.680054  4837 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [term 0 FOLLOWER]: Advancing to term 1
20:03:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.681149  4837 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 35eb0e66fe984442b2cb13b39c6a1a71 in term 1.
20:03:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.682039  5049 sys_catalog.cc:455] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } } }
20:03:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.682149  5049 sys_catalog.cc:458] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [sys.catalog]: This master's current role is: FOLLOWER
20:03:38.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.689734  5060 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:38.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.689831  5060 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:38.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.694383  4837 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [term 1 FOLLOWER]: Refusing update from remote peer 35eb0e66fe984442b2cb13b39c6a1a71: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.695042  5017 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [LEADER]: Connected to new peer: Peer: permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.696377  4910 raft_consensus.cc:1275] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [term 1 FOLLOWER]: Refusing update from remote peer 35eb0e66fe984442b2cb13b39c6a1a71: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.696905  5017 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.698658  4816 tablet.cc:2404] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.700484  5031 sys_catalog.cc:455] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 35eb0e66fe984442b2cb13b39c6a1a71. Latest consensus state: current_term: 1 leader_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } } }
20:03:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.700605  5031 sys_catalog.cc:458] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [sys.catalog]: This master's current role is: FOLLOWER
20:03:38.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.702925  5049 sys_catalog.cc:455] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 35eb0e66fe984442b2cb13b39c6a1a71. Latest consensus state: current_term: 1 leader_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } } }
20:03:38.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.703042  5049 sys_catalog.cc:458] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [sys.catalog]: This master's current role is: FOLLOWER
20:03:38.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.711342  5017 sys_catalog.cc:455] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } } }
20:03:38.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.711476  5017 sys_catalog.cc:458] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [sys.catalog]: This master's current role is: LEADER
20:03:38.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.711580  5017 sys_catalog.cc:455] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } } }
20:03:38.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.711643  5017 sys_catalog.cc:458] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71 [sys.catalog]: This master's current role is: LEADER
20:03:38.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.712357  5031 sys_catalog.cc:455] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } } }
20:03:38.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.712476  5031 sys_catalog.cc:458] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3 [sys.catalog]: This master's current role is: FOLLOWER
20:03:38.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.713074  5062 mvcc.cc:204] Tried to move back new op lower bound from 7287335398151565312 to 7287335398011469824. Current Snapshot: MvccSnapshot[applied={T|T < 7287335398151565312}]
20:03:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.714371  5049 sys_catalog.cc:455] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "991ed6bcbbeb454f870239f56067dfc8" member_type: VOTER last_known_addr { host: "127.4.172.126" port: 44523 } } peers { permanent_uuid: "fde830cced344bd3bb684650ace094f3" member_type: VOTER last_known_addr { host: "127.4.172.125" port: 33165 } } peers { permanent_uuid: "35eb0e66fe984442b2cb13b39c6a1a71" member_type: VOTER last_known_addr { host: "127.4.172.124" port: 33635 } } }
20:03:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.714480  5049 sys_catalog.cc:458] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8 [sys.catalog]: This master's current role is: FOLLOWER
20:03:38.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.715129  5048 mvcc.cc:204] Tried to move back new op lower bound from 7287335398151565312 to 7287335398011469824. Current Snapshot: MvccSnapshot[applied={T|T < 7287335398151565312}]
20:03:38.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.715824  5064 mvcc.cc:204] Tried to move back new op lower bound from 7287335398151565312 to 7287335398011469824. Current Snapshot: MvccSnapshot[applied={T|T < 7287335398151565312}]
20:03:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.721941  5046 catalog_manager.cc:1382] Generated new cluster ID: 3b28c211abea4625a5a8943f60b6c5e4
20:03:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.722018  5046 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
20:03:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.784873  5046 catalog_manager.cc:1405] Generated new certificate authority record
20:03:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.786566  5046 catalog_manager.cc:1539] Loading token signing keys...
20:03:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.801846  5046 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 35eb0e66fe984442b2cb13b39c6a1a71: Generated new TSK 0
20:03:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.803056  5046 catalog_manager.cc:1549] Initializing in-progress tserver states...
20:03:38.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.827196  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/logs
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.65:0
20:03:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.65
20:03:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.65
20:03:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:38.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.940971  5069 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:38.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.941184  5069 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:38.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.941342  5069 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:38.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.944859  5069 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:38.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.944988  5069 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.65
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.948954  5069 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.65:0
20:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.65
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5069
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.65
20:03:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/logs
20:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.950380  5069 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.951897  5069 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:38.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.958247  5074 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.958379  5069 server_base.cc:1061] running on GCE node
20:03:38.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.958424  5077 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:38.958274  5075 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:38.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.959342  5069 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:38.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.959813  5069 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:38.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.960963  5069 hybrid_clock.cc:648] HybridClock initialized: now 1779134618960950 us; error 30 us; skew 500 ppm
20:03:38.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.962682  5069 webserver.cc:533] Webserver started at http://127.4.172.65:43765/ using document root <none> and password file <none>
20:03:38.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.963236  5069 fs_manager.cc:362] Metadata directory not provided
20:03:38.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.963295  5069 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:38.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.963515  5069 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.965001  5069 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/instance:
20:03:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:03:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.965497  5069 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal/instance:
20:03:38.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:03:38.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.968858  5069 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.002s	sys 0.003s
20:03:38.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.970922  5083 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.971879  5069 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.971977  5069 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:38.972065  5069 fs_report.cc:389] FS layout report
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/data
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:39.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.003356  5069 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.004164  5069 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.004359  5069 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.004997  5069 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.006048  5069 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.006098  5069 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:03:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.006150  5069 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.006170  5069 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:03:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.014842  5069 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.65:46045
20:03:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.014873  5196 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.65:46045 every 8 connection(s)
20:03:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.015882  5069 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:03:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.016896  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 5069
20:03:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.016999  4785 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal/instance
20:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.020298  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data
20:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/logs
20:03:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:03:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.66:0
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.66
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.66
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.027124  5197 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:03:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.027166  5199 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:03:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.027413  5199 heartbeater.cc:461] Registering TS with master...
20:03:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.027412  5197 heartbeater.cc:461] Registering TS with master...
20:03:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.027439  5198 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:03:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.028110  5197 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:03:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.028220  5198 heartbeater.cc:461] Registering TS with master...
20:03:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.028390  5198 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:03:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.028110  5199 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:03:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.029721  4972 ts_manager.cc:194] Registered new tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:03:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.029827  4827 ts_manager.cc:194] Registered new tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:03:39.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.030890  4900 ts_manager.cc:194] Registered new tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:03:39.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.031116  4972 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.65:46057
20:03:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.127892  5204 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.128108  5204 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.128156  5204 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:39.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.131547  5204 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:39.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.131659  5204 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.66
20:03:39.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.135571  5204 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.66:0
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.66
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5204
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.66
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/logs
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.136669  5204 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.137713  5204 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.143942  5209 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.143945  5212 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.144316  5210 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.144315  5204 server_base.cc:1061] running on GCE node
20:03:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.144733  5204 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:39.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.145301  5204 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.146463  5204 hybrid_clock.cc:648] HybridClock initialized: now 1779134619146449 us; error 23 us; skew 500 ppm
20:03:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.148131  5204 webserver.cc:533] Webserver started at http://127.4.172.66:41487/ using document root <none> and password file <none>
20:03:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.148641  5204 fs_manager.cc:362] Metadata directory not provided
20:03:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.148690  5204 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.148839  5204 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:39.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.150331  5204 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/instance:
20:03:39.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fb2124f5c8a4a78b46d0db63c580371"
20:03:39.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:03:39.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.150822  5204 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal/instance:
20:03:39.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fb2124f5c8a4a78b46d0db63c580371"
20:03:39.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:03:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.153967  5204 fs_manager.cc:696] Time spent creating directory manager: real 0.003s	user 0.003s	sys 0.000s
20:03:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.156219  5218 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.157240  5204 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
20:03:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.157336  5204 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fb2124f5c8a4a78b46d0db63c580371"
20:03:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:03:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.157415  5204 fs_report.cc:389] FS layout report
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/data
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:39.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.172462  5204 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.173090  5204 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.173235  5204 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.173734  5204 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:39.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.174755  5204 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:39.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.174804  5204 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:03:39.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.174841  5204 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:39.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.174855  5204 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:03:39.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.184636  5204 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.66:44141
20:03:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.184664  5331 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.66:44141 every 8 connection(s)
20:03:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.185565  5204 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:03:39.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.189577  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 5204
20:03:39.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.189667  4785 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal/instance
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.193329  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/logs
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.67:0
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.67
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.67
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.195354  5333 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:03:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.195621  5333 heartbeater.cc:461] Registering TS with master...
20:03:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.196091  5334 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:03:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.196187  5334 heartbeater.cc:461] Registering TS with master...
20:03:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.196261  5333 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:03:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.196368  5334 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:03:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.196782  5332 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:03:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.197008  5332 heartbeater.cc:461] Registering TS with master...
20:03:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.197364  4827 ts_manager.cc:194] Registered new tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.197407  4900 ts_manager.cc:194] Registered new tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.197947  5332 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:03:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.198621  4972 ts_manager.cc:194] Registered new tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.199340  4972 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.66:33225
20:03:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.302176  5339 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.302399  5339 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.302441  5339 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:39.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.305843  5339 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:39.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.305954  5339 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.67
20:03:39.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.309809  5339 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.67:0
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.67
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5339
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.67
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/logs
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.311048  5339 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.312101  5339 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:39.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.318214  5347 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:39.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.318223  5345 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:39.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:39.318223  5344 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:39.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.318379  5339 server_base.cc:1061] running on GCE node
20:03:39.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.318986  5339 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:39.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.319517  5339 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:39.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.320660  5339 hybrid_clock.cc:648] HybridClock initialized: now 1779134619320644 us; error 36 us; skew 500 ppm
20:03:39.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.322422  5339 webserver.cc:533] Webserver started at http://127.4.172.67:39233/ using document root <none> and password file <none>
20:03:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.322973  5339 fs_manager.cc:362] Metadata directory not provided
20:03:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.323025  5339 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.323184  5339 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.324661  5339 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/instance:
20:03:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f37d275110584bbb8c3b91cf61249854"
20:03:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:03:39.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.325093  5339 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal/instance:
20:03:39.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f37d275110584bbb8c3b91cf61249854"
20:03:39.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:03:39.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.329332  5339 fs_manager.cc:696] Time spent creating directory manager: real 0.004s	user 0.006s	sys 0.000s
20:03:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.331974  5353 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.333211  5339 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.333310  5339 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f37d275110584bbb8c3b91cf61249854"
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.333384  5339 fs_report.cc:389] FS layout report
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/data
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.346290  5339 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:39.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.346940  5339 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:39.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.347084  5339 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:39.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.347589  5339 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:39.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.348587  5339 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:39.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.348634  5339 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
20:03:39.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.348673  5339 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:39.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.348686  5339 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
20:03:39.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.357175  5339 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.67:33409
20:03:39.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.357223  5466 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.67:33409 every 8 connection(s)
20:03:39.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.358192  5339 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:03:39.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.359279  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 5339
20:03:39.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.359376  4785 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal/instance
20:03:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.368129  5469 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:03:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.368201  5467 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:03:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.368228  5468 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:03:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.368371  5469 heartbeater.cc:461] Registering TS with master...
20:03:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.368368  5467 heartbeater.cc:461] Registering TS with master...
20:03:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.368368  5468 heartbeater.cc:461] Registering TS with master...
20:03:39.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.368964  5468 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:03:39.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.368974  5467 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:03:39.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.368978  5469 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:03:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.370025  4827 ts_manager.cc:194] Registered new tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.370016  4972 ts_manager.cc:194] Registered new tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.370138  4900 ts_manager.cc:194] Registered new tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.370642  4972 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.67:47331
20:03:39.381 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
20:03:39.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.380174  4785 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
20:03:39.383 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

20:03:39.392 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.4.172.126"
      port: 44523
    }
    bound_http_address {
      host: "127.4.172.126"
      port: 40013
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.4.172.125"
      port: 33165
    }
    bound_http_address {
      host: "127.4.172.125"
      port: 43677
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.4.172.124"
      port: 33635
    }
    bound_http_address {
      host: "127.4.172.124"
      port: 46587
    }
  }
}

20:03:39.395 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

20:03:39.397 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.4.172.65"
      port: 46045
    }
    bound_http_address {
      host: "127.4.172.65"
      port: 43765
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.4.172.66"
      port: 44141
    }
    bound_http_address {
      host: "127.4.172.66"
      port: 41487
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.4.172.67"
      port: 33409
    }
    bound_http_address {
      host: "127.4.172.67"
      port: 39233
    }
  }
}

20:03:39.397 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:03:39.556 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
20:03:39.620 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
20:03:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.653388  5042 catalog_manager.cc:1294] Loaded cluster ID: 3b28c211abea4625a5a8943f60b6c5e4
20:03:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.653481  5042 catalog_manager.cc:1587] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3: loading cluster ID for follower catalog manager: success
20:03:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.655807  5042 catalog_manager.cc:1609] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3: acquiring CA information for follower catalog manager: success
20:03:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.657475  5042 catalog_manager.cc:1637] T 00000000000000000000000000000000 P fde830cced344bd3bb684650ace094f3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.691447  5060 catalog_manager.cc:1294] Loaded cluster ID: 3b28c211abea4625a5a8943f60b6c5e4
20:03:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.691536  5060 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8: loading cluster ID for follower catalog manager: success
20:03:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.693964  5060 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8: acquiring CA information for follower catalog manager: success
20:03:39.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:39.696214  5060 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 991ed6bcbbeb454f870239f56067dfc8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:39.781 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.172.126:44523(127.4.172.126:44523)] connecting to peer
20:03:39.828 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.172.125:33165(127.4.172.125:33165)] connecting to peer
20:03:39.836 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.172.124:33635(127.4.172.124:33635)] connecting to peer
20:03:39.845 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.4.172.125:33165(127.4.172.125:33165)] Successfully connected to peer
20:03:39.846 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.172.126:44523(127.4.172.126:44523)] Successfully connected to peer
20:03:39.847 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.4.172.124:33635(127.4.172.124:33635)] Successfully connected to peer
20:03:39.931 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.4.172.126
20:03:39.931 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.4.172.124
20:03:39.938 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.4.172.125
20:03:40.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.032975  5197 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:03:40.152 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xdf493ab7, L:/127.0.0.1:53694 - R:/127.4.172.126:44523] using SASL/PLAIN
20:03:40.152 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x558ab875, L:/127.0.0.1:59070 - R:/127.4.172.124:33635] using SASL/PLAIN
20:03:40.152 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x74667d68, L:/127.0.0.1:41686 - R:/127.4.172.125:33165] using SASL/PLAIN
20:03:40.186 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
20:03:40.187 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=439), Traces: [0ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.172.126:44523, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.172.125:33165, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.172.124:33635, [414ms] Sub RPC ConnectToMaster: received response from server master-127.4.172.125:33165: OK, [414ms] Sub RPC ConnectToMaster: received response from server master-127.4.172.126:44523: OK, [414ms] Sub RPC ConnectToMaster: received response from server master-127.4.172.124:33635: OK, deferred=null) after lookup
20:03:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.200788  5332 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:03:40.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.230276  4972 catalog_manager.cc:2282] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59070:
20:03:40.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735"
20:03:40.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
20:03:40.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "key"
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: true
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column1_i"
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column2_i"
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: INT32
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column3_s"
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: STRING
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: true
20:03:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     encoding: DICT_ENCODING
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     compression: LZ4
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 4096
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   columns {
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     name: "column4_b"
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     type: BOOL
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_key: false
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_nullable: false
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     cfile_block_size: 0
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     immutable: false
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     is_auto_incrementing: false
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
20:03:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   range_schema {
20:03:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     columns {
20:03:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)       name: "key"
20:03:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)     }
20:03:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)   }
20:03:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
20:03:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:40.231513  4972 catalog_manager.cc:7058] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
20:03:40.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.253486  5131 tablet_service.cc:1511] Processing CreateTablet for tablet a42d6caf61bb47e19a3389753a1059a1 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735 [id=63ac054b69c24e59be09478d544244dd]), partition=RANGE (key) PARTITION UNBOUNDED
20:03:40.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.254364  5131 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a42d6caf61bb47e19a3389753a1059a1. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:40.262 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 63ac054b69c24e59be09478d544244dd
20:03:40.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.266350  5498 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Bootstrap starting.
20:03:40.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.269618  5498 tablet_bootstrap.cc:654] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Neither blocks nor log segments found. Creating new log.
20:03:40.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.269419  5401 tablet_service.cc:1511] Processing CreateTablet for tablet a42d6caf61bb47e19a3389753a1059a1 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735 [id=63ac054b69c24e59be09478d544244dd]), partition=RANGE (key) PARTITION UNBOUNDED
20:03:40.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.270058  5266 tablet_service.cc:1511] Processing CreateTablet for tablet a42d6caf61bb47e19a3389753a1059a1 (DEFAULT_TABLE table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735 [id=63ac054b69c24e59be09478d544244dd]), partition=RANGE (key) PARTITION UNBOUNDED
20:03:40.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.270484  5401 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a42d6caf61bb47e19a3389753a1059a1. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.271020  5266 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a42d6caf61bb47e19a3389753a1059a1. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.271035  5498 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Log is configured to *not* fsync() on all Append() calls
20:03:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.283473  5500 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap starting.
20:03:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.284458  5498 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: No bootstrap required, opened a new log
20:03:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.284699  5498 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Time spent bootstrapping tablet: real 0.019s	user 0.006s	sys 0.000s
20:03:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.284996  5501 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Bootstrap starting.
20:03:40.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.286217  5500 tablet_bootstrap.cc:654] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Neither blocks nor log segments found. Creating new log.
20:03:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.287110  5500 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Log is configured to *not* fsync() on all Append() calls
20:03:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.287181  5501 tablet_bootstrap.cc:654] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Neither blocks nor log segments found. Creating new log.
20:03:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.287917  5501 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Log is configured to *not* fsync() on all Append() calls
20:03:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.288700  5498 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:40.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.289677  5501 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: No bootstrap required, opened a new log
20:03:40.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.290082  5501 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Time spent bootstrapping tablet: real 0.005s	user 0.005s	sys 0.000s
20:03:40.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.290907  5498 raft_consensus.cc:385] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:40.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.291000  5498 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 533c3ef5bc9342ac84c55607e02629a8, State: Initialized, Role: FOLLOWER
20:03:40.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.291853  5498 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:40.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.292829  5500 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: No bootstrap required, opened a new log
20:03:40.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.292966  5498 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.000s
20:03:40.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.293293  5500 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Time spent bootstrapping tablet: real 0.010s	user 0.005s	sys 0.000s
20:03:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.294059  5501 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.294445  5501 raft_consensus.cc:385] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.294690  5501 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f37d275110584bbb8c3b91cf61249854, State: Initialized, Role: FOLLOWER
20:03:40.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.295358  5501 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:40.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.296396  5501 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.000s
20:03:40.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.296697  5500 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.296979  5467 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:03:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.297148  5500 raft_consensus.cc:385] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.297379  5500 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2fb2124f5c8a4a78b46d0db63c580371, State: Initialized, Role: FOLLOWER
20:03:40.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.298038  5500 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.299028  5500 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
20:03:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.311201  5506 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:40.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.312608  5506 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:40.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.313840  5506 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:40.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.318440  5151 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "2fb2124f5c8a4a78b46d0db63c580371" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "533c3ef5bc9342ac84c55607e02629a8" is_pre_election: true
20:03:40.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.318809  5151 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2fb2124f5c8a4a78b46d0db63c580371 in term 0.
20:03:40.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.319334  5421 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "2fb2124f5c8a4a78b46d0db63c580371" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f37d275110584bbb8c3b91cf61249854" is_pre_election: true
20:03:40.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.319525  5221 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, 533c3ef5bc9342ac84c55607e02629a8; no voters: 
20:03:40.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.319598  5421 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2fb2124f5c8a4a78b46d0db63c580371 in term 0.
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.320261  5506 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.320348  5506 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.320393  5506 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 0 FOLLOWER]: Advancing to term 1
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.321565  5506 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.321975  5506 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 1 election: Requested vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.322358  5151 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "2fb2124f5c8a4a78b46d0db63c580371" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:03:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.322489  5151 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 0 FOLLOWER]: Advancing to term 1
20:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.323017  5421 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "2fb2124f5c8a4a78b46d0db63c580371" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f37d275110584bbb8c3b91cf61249854"
20:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.323145  5421 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 0 FOLLOWER]: Advancing to term 1
20:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.323639  5151 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2fb2124f5c8a4a78b46d0db63c580371 in term 1.
20:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.324000  5221 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, 533c3ef5bc9342ac84c55607e02629a8; no voters: 
20:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.324221  5421 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2fb2124f5c8a4a78b46d0db63c580371 in term 1.
20:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.324796  5506 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 1 FOLLOWER]: Leader election won for term 1
20:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.325043  5506 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 1 LEADER]: Becoming Leader. State: Replica: 2fb2124f5c8a4a78b46d0db63c580371, State: Running, Role: LEADER
20:03:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.325403  5506 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:40.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.328792  4972 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66). New cstate: current_term: 1 leader_uuid: "2fb2124f5c8a4a78b46d0db63c580371" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: UNKNOWN } } }
20:03:40.354 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:894) Opened table 63ac054b69c24e59be09478d544244dd
20:03:40.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:40.361317  5470 tablet.cc:2404] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:40.374 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:40.381 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299993, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:40.382 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=19), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [9ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=null) after lookup
20:03:40.394 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:40.397 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] Successfully connected to peer
20:03:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:40.437794  5335 tablet.cc:2404] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:40.472 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xac03c271, L:/127.0.0.1:57298 - R:/127.4.172.66:44141] using TOKEN/null
20:03:40.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.481009  5151 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 1 FOLLOWER]: Refusing update from remote peer 2fb2124f5c8a4a78b46d0db63c580371: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:40.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.481009  5421 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 1 FOLLOWER]: Refusing update from remote peer 2fb2124f5c8a4a78b46d0db63c580371: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:40.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.481802  5506 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:40.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.482007  5509 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [LEADER]: Connected to new peer: Peer: permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:40.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.501226  5512 mvcc.cc:204] Tried to move back new op lower bound from 7287335405481734144 to 7287335404856250368. Current Snapshot: MvccSnapshot[applied={T|T < 7287335405481734144}]
20:03:40.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.504137  5513 mvcc.cc:204] Tried to move back new op lower bound from 7287335405481734144 to 7287335404856250368. Current Snapshot: MvccSnapshot[applied={T|T < 7287335405481734144}]
20:03:40.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.507021  5514 mvcc.cc:204] Tried to move back new op lower bound from 7287335405481734144 to 7287335404856250368. Current Snapshot: MvccSnapshot[applied={T|T < 7287335405481734144}]
20:03:40.620 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 3 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:40.626 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.4.172.66:44141
20:03:40.627 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:40.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:40.628297  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 5204
20:03:40.636 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:40.639 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] handling channelInactive
20:03:40.640 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state READY due to: connection closed
20:03:40.640 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: connection closed
20:03:40.652 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:40.653 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:40.653 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=2, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=16), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [4ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [16ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2076684398(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@4935e96e -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@252ef3ef -> wakeup thread Time-limited test)) after lookup
20:03:40.654 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:40.657 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.657 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.657 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.672 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:40.672 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:40.672 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=4, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=35), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [4ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [16ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [21ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [35ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2076684398(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@4935e96e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@252ef3ef -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226))) after lookup
20:03:40.673 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:40.675 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.675 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.675 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.691 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:40.692 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:40.692 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=6, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=55), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [4ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [16ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [21ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [35ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [37ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [39ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [39ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [55ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2076684398(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@4935e96e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226) -> (continuation of Deferred@1543051574 after retry RPC@157780435), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@252ef3ef -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226) -> (continuation of Deferred@1543051574 after retry RPC@157780435))) after lookup
20:03:40.693 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:40.695 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.695 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.695 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.712 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:40.712 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:40.713 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=8, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=76), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [4ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [16ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [21ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [35ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [37ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [39ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [39ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [55ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [57ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [59ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [59ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [76ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2076684398(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@4935e96e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226) -> (continuation of Deferred@1543051574 after retry RPC@157780435) -> (continuation of Deferred@966305589 after retry RPC@966076210), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@252ef3ef -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226) -> (continuation of Deferred@1543051574 after retry RPC@157780435) -> (continuation of Deferred@966305589 after retry RPC@966076210))) after lookup
20:03:40.714 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:40.715 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.715 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:40.716 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:41.012 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:41.012 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:41.013 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=10, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=376), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [4ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [16ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [21ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [35ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [37ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [39ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [39ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [55ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [57ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [59ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [59ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [76ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [77ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [79ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [80ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [372ms] refreshing cache from master, [373ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [375ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2076684398(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@4935e96e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226) -> (continuation of Deferred@1543051574 after retry RPC@157780435) -> (continuation of Deferred@966305589 after retry RPC@966076210) -> (continuation of Deferred@1510714203 after retry RPC@1787926881), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@252ef3ef -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226) -> (continuation of Deferred@1543051574 after retry RPC@157780435) -> (continuation of Deferred@966305589 after retry RPC@966076210) -> (continuation of Deferred@1510714203 after retry RPC@1787926881))) after lookup
20:03:41.014 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:41.015 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:41.015 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:41.015 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:41.052 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:41.052 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:41.053 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=12, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=416), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [4ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [16ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [21ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [35ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [37ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [39ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [39ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [55ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [57ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [59ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [59ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [76ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [77ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [79ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [80ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [372ms] refreshing cache from master, [373ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [375ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [377ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [379ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [380ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [412ms] refreshing cache from master, [412ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [415ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2076684398(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@4935e96e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226) -> (continuation of Deferred@1543051574 after retry RPC@157780435) -> (continuation of Deferred@966305589 after retry RPC@966076210) -> (continuation of Deferred@1510714203 after retry RPC@1787926881) -> (continuation of Deferred@2085364381 after retry RPC@451999460), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@252ef3ef -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226) -> (continuation of Deferred@1543051574 after retry RPC@157780435) -> (continuation of Deferred@966305589 after retry RPC@966076210) -> (continuation of Deferred@1510714203 after retry RPC@1787926881) -> (continuation of Deferred@2085364381 after retry RPC@451999460))) after lookup
20:03:41.054 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:41.055 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:41.055 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:41.055 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.041695  5527 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2fb2124f5c8a4a78b46d0db63c580371)
20:03:42.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.041838  5527 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:42.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.042508  5528 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2fb2124f5c8a4a78b46d0db63c580371)
20:03:42.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.042655  5528 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:42.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.042979  5527 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:42.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.043761  5528 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:42.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.044201  5086 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:03:42.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.044281  5356 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:03:42.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.045070  5356 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:03:42.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.047684  5151 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "f37d275110584bbb8c3b91cf61249854" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "533c3ef5bc9342ac84c55607e02629a8" is_pre_election: true
20:03:42.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.047838  5151 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f37d275110584bbb8c3b91cf61249854 in term 1.
20:03:42.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.048233  5356 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 533c3ef5bc9342ac84c55607e02629a8, f37d275110584bbb8c3b91cf61249854; no voters: 2fb2124f5c8a4a78b46d0db63c580371
20:03:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.048547  5527 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:03:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.048610  5527 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 2fb2124f5c8a4a78b46d0db63c580371)
20:03:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.048651  5527 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 1 FOLLOWER]: Advancing to term 2
20:03:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.048743  5086 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:03:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.048909  5421 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "533c3ef5bc9342ac84c55607e02629a8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f37d275110584bbb8c3b91cf61249854" is_pre_election: true
20:03:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.049480  5527 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.049710  5421 raft_consensus.cc:2393] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 533c3ef5bc9342ac84c55607e02629a8 in current term 2: Already voted for candidate f37d275110584bbb8c3b91cf61249854 in this term.
20:03:42.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.049811  5527 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 2 election: Requested vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:42.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.050236  5151 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "f37d275110584bbb8c3b91cf61249854" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:03:42.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.050352  5151 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 1 FOLLOWER]: Advancing to term 2
20:03:42.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.050405  5087 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 533c3ef5bc9342ac84c55607e02629a8; no voters: 2fb2124f5c8a4a78b46d0db63c580371, f37d275110584bbb8c3b91cf61249854
20:03:42.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.051277  5151 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f37d275110584bbb8c3b91cf61249854 in term 2.
20:03:42.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.051453  5528 raft_consensus.cc:2749] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:03:42.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.051514  5356 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:03:42.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.051640  5356 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 533c3ef5bc9342ac84c55607e02629a8, f37d275110584bbb8c3b91cf61249854; no voters: 2fb2124f5c8a4a78b46d0db63c580371
20:03:42.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.051766  5527 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 2 FOLLOWER]: Leader election won for term 2
20:03:42.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.051951  5527 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 2 LEADER]: Becoming Leader. State: Replica: f37d275110584bbb8c3b91cf61249854, State: Running, Role: LEADER
20:03:42.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.052243  5527 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.054996  4972 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 reported cstate change: term changed from 1 to 2, leader changed from 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66) to f37d275110584bbb8c3b91cf61249854 (127.4.172.67). New cstate: current_term: 2 leader_uuid: "f37d275110584bbb8c3b91cf61249854" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: HEALTHY } } }
20:03:42.232 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:42.232 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:42.233 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=14, sequence_id=4, TimeoutTracker(timeout=30000, elapsed=1596), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [4ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: connection closed, [4ms] delaying RPC due to: Network error: connection closed, [13ms] refreshing cache from master, [13ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [16ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [21ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [32ms] refreshing cache from master, [33ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [35ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [37ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [39ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [39ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [52ms] refreshing cache from master, [52ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [55ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [57ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [59ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [59ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [73ms] refreshing cache from master, [73ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [76ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [77ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [79ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [80ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [372ms] refreshing cache from master, [373ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [375ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [377ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [379ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [380ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [412ms] refreshing cache from master, [412ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [415ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [417ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [419ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [419ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [1593ms] refreshing cache from master, [1593ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [1596ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2076684398(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@4935e96e -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226) -> (continuation of Deferred@1543051574 after retry RPC@157780435) -> (continuation of Deferred@966305589 after retry RPC@966076210) -> (continuation of Deferred@1510714203 after retry RPC@1787926881) -> (continuation of Deferred@2085364381 after retry RPC@451999460) -> (continuation of Deferred@271969916 after retry RPC@233794888), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@252ef3ef -> wakeup thread Time-limited test -> (continuation of Deferred@897863281 after retry RPC@441298226) -> (continuation of Deferred@1543051574 after retry RPC@157780435) -> (continuation of Deferred@966305589 after retry RPC@966076210) -> (continuation of Deferred@1510714203 after retry RPC@1787926881) -> (continuation of Deferred@2085364381 after retry RPC@451999460) -> (continuation of Deferred@271969916 after retry RPC@233794888))) after lookup
20:03:42.234 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:42.235 [DEBUG - kudu-nio-2] (Connection.java:757) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] Successfully connected to peer
20:03:42.256 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xfff85277, L:/127.0.0.1:55862 - R:/127.4.172.67:33409] using TOKEN/null
20:03:42.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.261041  5151 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 2 FOLLOWER]: Refusing update from remote peer f37d275110584bbb8c3b91cf61249854: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
20:03:42.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.261444  5356 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 -> Peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Couldn't send request to peer 2fb2124f5c8a4a78b46d0db63c580371. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:42.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.261700  5537 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [LEADER]: Connected to new peer: Peer: permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
20:03:42.284 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.4.172.66:44141
20:03:42.285 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.286226  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/logs
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:42.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.66:44141
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.66
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41487
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.66
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:42.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.390800  5555 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:42.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.391394  5555 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:42.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.391636  5555 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:42.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.395689  5555 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:42.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.395817  5555 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.66
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.399866  5555 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.66:44141
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.66
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41487
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5555
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.66
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/logs
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.401407  5555 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:42.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.402622  5555 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:42.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.409103  5563 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:42.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.409093  5560 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:42.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:42.409093  5561 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:42.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.409372  5555 server_base.cc:1061] running on GCE node
20:03:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.410065  5555 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.410724  5555 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:42.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.411921  5555 hybrid_clock.cc:648] HybridClock initialized: now 1779134622411875 us; error 61 us; skew 500 ppm
20:03:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.413934  5555 webserver.cc:533] Webserver started at http://127.4.172.66:41487/ using document root <none> and password file <none>
20:03:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.414511  5555 fs_manager.cc:362] Metadata directory not provided
20:03:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.414633  5555 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:42.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.418103  5555 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
20:03:42.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.420208  5569 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:42.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.421293  5555 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.001s	sys 0.000s
20:03:42.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.421411  5555 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fb2124f5c8a4a78b46d0db63c580371"
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.421877  5555 fs_report.cc:389] FS layout report
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/data
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.447203  5555 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.447988  5555 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.448179  5555 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.448854  5555 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.450238  5576 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
20:03:42.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.453729  5555 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.453807  5555 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
20:03:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.453877  5555 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.455572  5555 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.455621  5555 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
20:03:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.455955  5576 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap starting.
20:03:42.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.466382  5555 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.66:44141
20:03:42.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.466449  5685 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.66:44141 every 8 connection(s)
20:03:42.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.467463  5555 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:03:42.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.472015  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 5555
20:03:42.472 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:42.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.472764  5576 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Log is configured to *not* fsync() on all Append() calls
20:03:42.481 [DEBUG - kudu-nio-2] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 6 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:42.482 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.4.172.67:33409
20:03:42.483 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.483333  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 5339
20:03:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.484863  5576 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.485086  5687 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:03:42.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.485334  5687 heartbeater.cc:461] Registering TS with master...
20:03:42.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.485436  5576 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap complete.
20:03:42.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.485904  5687 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:03:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.486346  5686 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:03:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.486442  5686 heartbeater.cc:461] Registering TS with master...
20:03:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.486555  5576 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Time spent bootstrapping tablet: real 0.031s	user 0.013s	sys 0.012s
20:03:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.486647  5686 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:03:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.486370  5688 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:03:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.486799  5688 heartbeater.cc:461] Registering TS with master...
20:03:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.486977  5688 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:03:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.488091  4826 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.488772  4900 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.489048  5576 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.489379  4972 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.489424  5576 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2fb2124f5c8a4a78b46d0db63c580371, State: Initialized, Role: FOLLOWER
20:03:42.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.489846  5576 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:42.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.490293  4972 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.66:38329
20:03:42.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.490759  5576 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Time spent starting tablet: real 0.004s	user 0.002s	sys 0.002s
20:03:42.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:42.492043  5686 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:03:42.493 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:42.493 [DEBUG - kudu-nio-2] (Connection.java:239) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] handling channelInactive
20:03:42.493 [DEBUG - kudu-nio-2] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state READY due to: connection closed
20:03:42.494 [DEBUG - Time-limited test] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:42.495 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.495 [DEBUG - kudu-nio-3] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.495 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.511 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:42.511 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:42.512 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=2, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=18), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [18ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1722609472(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@65c3613 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dac7439 -> wakeup thread Time-limited test)) after lookup
20:03:42.513 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:42.514 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.514 [DEBUG - kudu-nio-4] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.514 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.531 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:42.531 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:42.532 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=4, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=38), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [18ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [20ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [21ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [21ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [38ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1722609472(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@65c3613 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1127061749 after retry RPC@233709845), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dac7439 -> wakeup thread Time-limited test -> (continuation of Deferred@1127061749 after retry RPC@233709845))) after lookup
20:03:42.532 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:42.533 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.533 [DEBUG - kudu-nio-5] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.533 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.572 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:42.572 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:42.573 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=6, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=79), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [18ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [20ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [21ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [21ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [38ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [39ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [40ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [41ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [79ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1722609472(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@65c3613 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1127061749 after retry RPC@233709845) -> (continuation of Deferred@447508993 after retry RPC@1723207368), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dac7439 -> wakeup thread Time-limited test -> (continuation of Deferred@1127061749 after retry RPC@233709845) -> (continuation of Deferred@447508993 after retry RPC@1723207368))) after lookup
20:03:42.574 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:42.575 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.576 [DEBUG - kudu-nio-6] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.576 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.671 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:42.671 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:42.672 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=8, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=178), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [18ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [20ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [21ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [21ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [38ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [39ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [40ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [41ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [79ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [80ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [83ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [83ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [178ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1722609472(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@65c3613 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1127061749 after retry RPC@233709845) -> (continuation of Deferred@447508993 after retry RPC@1723207368) -> (continuation of Deferred@450891708 after retry RPC@1291223654), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dac7439 -> wakeup thread Time-limited test -> (continuation of Deferred@1127061749 after retry RPC@233709845) -> (continuation of Deferred@447508993 after retry RPC@1723207368) -> (continuation of Deferred@450891708 after retry RPC@1291223654))) after lookup
20:03:42.672 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:42.674 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.674 [DEBUG - kudu-nio-7] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:42.674 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:43.132 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:43.132 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:43.132 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=10, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=639), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [18ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [20ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [21ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [21ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [38ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [39ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [40ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [41ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [79ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [80ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [83ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [83ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [178ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [179ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [181ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [181ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [636ms] refreshing cache from master, [636ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [638ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1722609472(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@65c3613 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1127061749 after retry RPC@233709845) -> (continuation of Deferred@447508993 after retry RPC@1723207368) -> (continuation of Deferred@450891708 after retry RPC@1291223654) -> (continuation of Deferred@655926902 after retry RPC@39657033), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dac7439 -> wakeup thread Time-limited test -> (continuation of Deferred@1127061749 after retry RPC@233709845) -> (continuation of Deferred@447508993 after retry RPC@1723207368) -> (continuation of Deferred@450891708 after retry RPC@1291223654) -> (continuation of Deferred@655926902 after retry RPC@39657033))) after lookup
20:03:43.133 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:43.134 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:43.134 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:43.135 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.784231  5699 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader f37d275110584bbb8c3b91cf61249854)
20:03:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.784368  5699 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.784785  5699 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:43.786119  5087 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409): Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111)
20:03:43.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.789620  5638 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "533c3ef5bc9342ac84c55607e02629a8" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "2fb2124f5c8a4a78b46d0db63c580371" is_pre_election: true
20:03:43.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.789896  5638 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 533c3ef5bc9342ac84c55607e02629a8 in term 1.
20:03:43.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.790597  5086 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, 533c3ef5bc9342ac84c55607e02629a8; no voters: f37d275110584bbb8c3b91cf61249854
20:03:43.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.790813  5699 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 2 FOLLOWER]: Leader pre-election won for term 3
20:03:43.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.790887  5699 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 2 FOLLOWER]: Starting leader election (detected failure of leader f37d275110584bbb8c3b91cf61249854)
20:03:43.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.790933  5699 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 2 FOLLOWER]: Advancing to term 3
20:03:43.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.791782  5699 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:43.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.792156  5699 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 3 election: Requested vote from peers 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:43.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.792534  5638 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "533c3ef5bc9342ac84c55607e02629a8" candidate_term: 3 candidate_status { last_received { term: 2 index: 8 } } ignore_live_leader: false dest_uuid: "2fb2124f5c8a4a78b46d0db63c580371"
20:03:43.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.792673  5638 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 1 FOLLOWER]: Advancing to term 3
20:03:43.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:43.793134  5087 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409): Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111)
20:03:43.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.794597  5638 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 533c3ef5bc9342ac84c55607e02629a8 in term 3.
20:03:43.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.794934  5086 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, 533c3ef5bc9342ac84c55607e02629a8; no voters: f37d275110584bbb8c3b91cf61249854
20:03:43.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.795120  5699 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 3 FOLLOWER]: Leader election won for term 3
20:03:43.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.795362  5699 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 3 LEADER]: Becoming Leader. State: Replica: 533c3ef5bc9342ac84c55607e02629a8, State: Running, Role: LEADER
20:03:43.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.795643  5699 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:43.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:43.798102  4972 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 reported cstate change: term changed from 2 to 3, leader changed from f37d275110584bbb8c3b91cf61249854 (127.4.172.67) to 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65). New cstate: current_term: 3 leader_uuid: "533c3ef5bc9342ac84c55607e02629a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: UNKNOWN } } }
20:03:44.032 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:44.032 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:44.033 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=12, sequence_id=7, TimeoutTracker(timeout=30000, elapsed=1539), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [15ms] refreshing cache from master, [16ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [18ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [20ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [21ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [21ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [35ms] refreshing cache from master, [35ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [38ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [39ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [40ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [41ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [76ms] refreshing cache from master, [76ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [79ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [80ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [83ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [83ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [175ms] refreshing cache from master, [175ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [178ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [179ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [181ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [181ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [636ms] refreshing cache from master, [636ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [638ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [640ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [641ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [642ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [1536ms] refreshing cache from master, [1536ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [1539ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1722609472(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@65c3613 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1127061749 after retry RPC@233709845) -> (continuation of Deferred@447508993 after retry RPC@1723207368) -> (continuation of Deferred@450891708 after retry RPC@1291223654) -> (continuation of Deferred@655926902 after retry RPC@39657033) -> (continuation of Deferred@1922584493 after retry RPC@408795361), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@2dac7439 -> wakeup thread Time-limited test -> (continuation of Deferred@1127061749 after retry RPC@233709845) -> (continuation of Deferred@447508993 after retry RPC@1723207368) -> (continuation of Deferred@450891708 after retry RPC@1291223654) -> (continuation of Deferred@655926902 after retry RPC@39657033) -> (continuation of Deferred@1922584493 after retry RPC@408795361))) after lookup
20:03:44.034 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:44.035 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] Successfully connected to peer
20:03:44.049 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc840b4da, L:/127.0.0.1:47262 - R:/127.4.172.65:46045] using TOKEN/null
20:03:44.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.053424  5638 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 3 FOLLOWER]: Refusing update from remote peer 533c3ef5bc9342ac84c55607e02629a8: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
20:03:44.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:44.053544  5087 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 -> Peer f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409): Couldn't send request to peer f37d275110584bbb8c3b91cf61249854. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:44.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.054091  5704 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 4, Time since last communication: 0.000s
20:03:44.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.060240  5720 mvcc.cc:204] Tried to move back new op lower bound from 7287335420116168704 to 7287335419069591552. Current Snapshot: MvccSnapshot[applied={T|T < 7287335420116168704}]
20:03:44.076 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.4.172.67:33409
20:03:44.077 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.077639  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data
20:03:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/logs
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.67:33409
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.67
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39233
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.67
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:44.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:44.179839  5727 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:44.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:44.180109  5727 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:44.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:44.180187  5727 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:44.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:44.183651  5727 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:44.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:44.183763  5727 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.67
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.189090  5727 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.67:33409
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.67
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39233
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5727
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.67
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/logs
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:44.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.190563  5727 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:44.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.191991  5727 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:44.199533  5733 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:44.199561  5732 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.199635  5727 server_base.cc:1061] running on GCE node
20:03:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:44.199565  5735 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.200176  5727 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.200670  5727 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:44.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.201830  5727 hybrid_clock.cc:648] HybridClock initialized: now 1779134624201810 us; error 31 us; skew 500 ppm
20:03:44.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.203828  5727 webserver.cc:533] Webserver started at http://127.4.172.67:39233/ using document root <none> and password file <none>
20:03:44.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.204461  5727 fs_manager.cc:362] Metadata directory not provided
20:03:44.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.204541  5727 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:44.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.208401  5727 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.004s	sys 0.000s
20:03:44.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.210536  5741 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.211623  5727 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.003s	sys 0.000s
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.211748  5727 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f37d275110584bbb8c3b91cf61249854"
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.212235  5727 fs_report.cc:389] FS layout report
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/data
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.233693  5727 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.234346  5727 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.234503  5727 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:44.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.235275  5727 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:44.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.236606  5748 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
20:03:44.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.240002  5727 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:44.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.240070  5727 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
20:03:44.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.240139  5727 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:44.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.242035  5727 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:44.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.242277  5727 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.002s	sys 0.000s
20:03:44.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.242444  5748 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Bootstrap starting.
20:03:44.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.252357  5727 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.67:33409
20:03:44.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.252377  5855 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.67:33409 every 8 connection(s)
20:03:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.253367  5727 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:03:44.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.257545  5748 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Log is configured to *not* fsync() on all Append() calls
20:03:44.263 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:44.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.263273  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 5727
20:03:44.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.264099  5858 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:03:44.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.264339  5858 heartbeater.cc:461] Registering TS with master...
20:03:44.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.264882  5858 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:03:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.266570  5856 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.266664  5856 heartbeater.cc:461] Registering TS with master...
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.266858  5856 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.266983  4897 ts_manager.cc:194] Re-registered known tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.267752  4971 ts_manager.cc:194] Re-registered known tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.267848  5857 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.267920  5857 heartbeater.cc:461] Registering TS with master...
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.268093  5857 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.268568  5748 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=6 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.268728  4971 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.67:44035
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.268925  5748 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Bootstrap complete.
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.268949  4826 ts_manager.cc:194] Re-registered known tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.269898  5748 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Time spent bootstrapping tablet: real 0.028s	user 0.020s	sys 0.004s
20:03:44.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.271998  5748 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:44.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.272387  5748 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: f37d275110584bbb8c3b91cf61249854, State: Initialized, Role: FOLLOWER
20:03:44.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.272891  5748 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:44.273 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 9 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:44.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.273765  5748 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Time spent starting tablet: real 0.004s	user 0.007s	sys 0.000s
20:03:44.274 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.4.172.65:46045
20:03:44.274 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.274573  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 5069
20:03:44.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:44.277439  5856 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:03:44.281 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] handling channelInactive
20:03:44.282 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state READY due to: connection closed
20:03:44.282 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:44.283 [DEBUG - Time-limited test] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:44.284 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.284 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.284 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.290 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:44.291 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:44.291 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=2, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=8), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [7ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@668626250(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6487aa53 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@39db4354 -> wakeup thread Time-limited test)) after lookup
20:03:44.291 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:44.292 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.292 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.292 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.311 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:44.311 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:44.312 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=4, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [7ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [8ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [9ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [9ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@668626250(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6487aa53 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@39db4354 -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137))) after lookup
20:03:44.313 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:44.314 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.314 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.314 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.332 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:44.332 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:44.332 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=6, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [7ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [8ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [9ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [9ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [29ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [31ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [31ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [48ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@668626250(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6487aa53 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137) -> (continuation of Deferred@826272663 after retry RPC@77969898), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@39db4354 -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137) -> (continuation of Deferred@826272663 after retry RPC@77969898))) after lookup
20:03:44.333 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:44.334 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.334 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.334 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.391 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:44.391 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:44.391 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=8, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=108), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [7ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [8ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [9ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [9ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [29ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [31ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [31ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [48ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [50ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [51ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [51ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [107ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@668626250(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6487aa53 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137) -> (continuation of Deferred@826272663 after retry RPC@77969898) -> (continuation of Deferred@1280361984 after retry RPC@1704156297), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@39db4354 -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137) -> (continuation of Deferred@826272663 after retry RPC@77969898) -> (continuation of Deferred@1280361984 after retry RPC@1704156297))) after lookup
20:03:44.392 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:44.393 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.393 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.393 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.771 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:44.771 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:44.771 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=10, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=488), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [7ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [8ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [9ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [9ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [29ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [31ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [31ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [48ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [50ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [51ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [51ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [107ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [109ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [110ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [110ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [487ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@668626250(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6487aa53 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137) -> (continuation of Deferred@826272663 after retry RPC@77969898) -> (continuation of Deferred@1280361984 after retry RPC@1704156297) -> (continuation of Deferred@1594054208 after retry RPC@545992253), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@39db4354 -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137) -> (continuation of Deferred@826272663 after retry RPC@77969898) -> (continuation of Deferred@1280361984 after retry RPC@1704156297) -> (continuation of Deferred@1594054208 after retry RPC@545992253))) after lookup
20:03:44.772 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:44.773 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.773 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.773 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.812 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:44.812 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:44.812 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=12, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=529), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [7ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [8ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [9ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [9ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [29ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [31ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [31ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [48ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [50ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [51ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [51ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [107ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [109ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [110ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [110ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [487ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [489ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [490ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [491ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [526ms] refreshing cache from master, [526ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [529ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@668626250(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6487aa53 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137) -> (continuation of Deferred@826272663 after retry RPC@77969898) -> (continuation of Deferred@1280361984 after retry RPC@1704156297) -> (continuation of Deferred@1594054208 after retry RPC@545992253) -> (continuation of Deferred@802224922 after retry RPC@1448333851), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@39db4354 -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137) -> (continuation of Deferred@826272663 after retry RPC@77969898) -> (continuation of Deferred@1280361984 after retry RPC@1704156297) -> (continuation of Deferred@1594054208 after retry RPC@545992253) -> (continuation of Deferred@802224922 after retry RPC@1448333851))) after lookup
20:03:44.813 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:44.814 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.814 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:44.814 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:45.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.575790  5869 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 533c3ef5bc9342ac84c55607e02629a8)
20:03:45.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.575933  5869 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.576987  5869 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:45.577669  5572 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111)
20:03:45.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:45.578416  5572 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111)
20:03:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.582352  5810 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "2fb2124f5c8a4a78b46d0db63c580371" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "f37d275110584bbb8c3b91cf61249854" is_pre_election: true
20:03:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.582675  5810 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2fb2124f5c8a4a78b46d0db63c580371 in term 2.
20:03:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.583369  5573 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, f37d275110584bbb8c3b91cf61249854; no voters: 533c3ef5bc9342ac84c55607e02629a8
20:03:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.583639  5869 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 3 FOLLOWER]: Leader pre-election won for term 4
20:03:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.583703  5869 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 3 FOLLOWER]: Starting leader election (detected failure of leader 533c3ef5bc9342ac84c55607e02629a8)
20:03:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.583731  5869 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 3 FOLLOWER]: Advancing to term 4
20:03:45.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.584506  5869 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:45.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.584867  5869 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 4 election: Requested vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:45.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.585289  5810 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "2fb2124f5c8a4a78b46d0db63c580371" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "f37d275110584bbb8c3b91cf61249854"
20:03:45.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.585474  5810 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 2 FOLLOWER]: Advancing to term 4
20:03:45.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:45.585937  5572 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111)
20:03:45.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.587335  5810 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2fb2124f5c8a4a78b46d0db63c580371 in term 4.
20:03:45.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.587643  5573 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, f37d275110584bbb8c3b91cf61249854; no voters: 533c3ef5bc9342ac84c55607e02629a8
20:03:45.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.587862  5869 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 4 FOLLOWER]: Leader election won for term 4
20:03:45.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.588065  5869 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 4 LEADER]: Becoming Leader. State: Replica: 2fb2124f5c8a4a78b46d0db63c580371, State: Running, Role: LEADER
20:03:45.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.588372  5869 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:45.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:45.590956  4971 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 reported cstate change: term changed from 3 to 4, leader changed from 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65) to 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66). New cstate: current_term: 4 leader_uuid: "2fb2124f5c8a4a78b46d0db63c580371" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: UNKNOWN } } }
20:03:46.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:46.121173  5572 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 -> Peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Couldn't send request to peer 533c3ef5bc9342ac84c55607e02629a8. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:46.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:46.122056  5810 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 4 FOLLOWER]: Refusing update from remote peer 2fb2124f5c8a4a78b46d0db63c580371: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
20:03:46.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:46.122737  5887 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 8, Time since last communication: 0.000s
20:03:48.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:48.647799  5572 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 -> Peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Couldn't send request to peer 533c3ef5bc9342ac84c55607e02629a8. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:48.752 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:48.752 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:48.753 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=14, sequence_id=10, TimeoutTracker(timeout=30000, elapsed=4469), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [5ms] refreshing cache from master, [5ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [7ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [8ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [9ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [9ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [29ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [31ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [31ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [45ms] refreshing cache from master, [45ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [48ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [50ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [51ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [51ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [105ms] refreshing cache from master, [105ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [107ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [109ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [110ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [110ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [485ms] refreshing cache from master, [485ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [487ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [489ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [490ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [491ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [526ms] refreshing cache from master, [526ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [529ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [530ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [531ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [531ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [4466ms] refreshing cache from master, [4466ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [4469ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@668626250(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6487aa53 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137) -> (continuation of Deferred@826272663 after retry RPC@77969898) -> (continuation of Deferred@1280361984 after retry RPC@1704156297) -> (continuation of Deferred@1594054208 after retry RPC@545992253) -> (continuation of Deferred@802224922 after retry RPC@1448333851) -> (continuation of Deferred@855374369 after retry RPC@2054174063), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@39db4354 -> wakeup thread Time-limited test -> (continuation of Deferred@1167083956 after retry RPC@649225137) -> (continuation of Deferred@826272663 after retry RPC@77969898) -> (continuation of Deferred@1280361984 after retry RPC@1704156297) -> (continuation of Deferred@1594054208 after retry RPC@545992253) -> (continuation of Deferred@802224922 after retry RPC@1448333851) -> (continuation of Deferred@855374369 after retry RPC@2054174063))) after lookup
20:03:48.754 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:48.755 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] Successfully connected to peer
20:03:48.773 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2819d9cd, L:/127.0.0.1:57374 - R:/127.4.172.66:44141] using TOKEN/null
20:03:48.791 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.4.172.65:46045
20:03:48.791 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.792305  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/logs
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.65:46045
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.65
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43765
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.65
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:48.892153  5913 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:48.892372  5913 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:48.892428  5913 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:48.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:48.895787  5913 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:48.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:48.895905  5913 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.65
20:03:48.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.899710  5913 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:48.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.65:46045
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.65
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43765
20:03:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.5913
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.65
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/logs
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.901080  5913 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:48.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.902374  5913 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:48.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:48.908774  5919 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:48.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:48.908768  5918 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:48.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:48.908841  5921 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:48.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.908921  5913 server_base.cc:1061] running on GCE node
20:03:48.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.909641  5913 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:48.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.910205  5913 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:48.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.911351  5913 hybrid_clock.cc:648] HybridClock initialized: now 1779134628911335 us; error 27 us; skew 500 ppm
20:03:48.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.913098  5913 webserver.cc:533] Webserver started at http://127.4.172.65:43765/ using document root <none> and password file <none>
20:03:48.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.913686  5913 fs_manager.cc:362] Metadata directory not provided
20:03:48.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.913761  5913 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:48.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.917250  5913 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
20:03:48.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.919054  5927 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.919976  5913 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.001s
20:03:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.920080  5913 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:03:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:03:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:03:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.920514  5913 fs_report.cc:389] FS layout report
20:03:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:03:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:03:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/data
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:48.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.930166  5913 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:48.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.930930  5913 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:48.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.931113  5913 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:48.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.931676  5913 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:48.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.932905  5934 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
20:03:48.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.936143  5913 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:48.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.936205  5913 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
20:03:48.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.936266  5913 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:48.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.937775  5913 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.937817  5913 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
20:03:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.938102  5934 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Bootstrap starting.
20:03:48.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.948125  5913 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.65:46045
20:03:48.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.948155  6041 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.65:46045 every 8 connection(s)
20:03:48.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.949064  5913 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:03:48.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.953441  5934 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Log is configured to *not* fsync() on all Append() calls
20:03:48.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.957516  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 5913
20:03:48.957 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:48.965 [DEBUG - kudu-nio-1] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 12 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.964785  5934 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=9 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.965150  5934 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Bootstrap complete.
20:03:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.965268  6043 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:03:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.965485  6043 heartbeater.cc:461] Registering TS with master...
20:03:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.966009  6043 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:03:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.966239  5934 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Time spent bootstrapping tablet: real 0.028s	user 0.016s	sys 0.008s
20:03:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.968597  4827 ts_manager.cc:194] Re-registered known tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:03:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.968657  5934 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:48.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.969043  5934 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 533c3ef5bc9342ac84c55607e02629a8, State: Initialized, Role: FOLLOWER
20:03:48.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.969527  5934 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:48.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.970376  5934 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Time spent starting tablet: real 0.004s	user 0.007s	sys 0.000s
20:03:48.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.972043  6044 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:03:48.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.972138  6044 heartbeater.cc:461] Registering TS with master...
20:03:48.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.972363  6044 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:03:48.973 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.4.172.66:44141
20:03:48.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.973196  4897 ts_manager.cc:194] Re-registered known tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:03:48.973 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:48.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.974448  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 5555
20:03:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.975769  6042 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:03:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.975847  6042 heartbeater.cc:461] Registering TS with master...
20:03:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.976018  6042 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:03:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.976732  4971 ts_manager.cc:194] Re-registered known tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:03:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.977654  4971 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.65:35901
20:03:48.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:48.979808  6042 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:03:48.982 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:48.982 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] handling channelInactive
20:03:48.982 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state READY due to: connection closed
20:03:48.988 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:48.990 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:48.990 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:48.990 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.011 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:49.011 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:49.011 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=3, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=29), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [8ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@299475233(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@29da3de6 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@421c329a -> wakeup thread Time-limited test)) after lookup
20:03:49.012 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:49.013 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.013 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.013 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.031 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:49.032 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:49.032 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=5, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=49), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [8ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [30ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [31ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [31ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [49ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@299475233(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@29da3de6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1913620629 after retry RPC@1369767084), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@421c329a -> wakeup thread Time-limited test -> (continuation of Deferred@1913620629 after retry RPC@1369767084))) after lookup
20:03:49.033 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:49.034 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.034 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.034 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.091 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:49.091 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:49.092 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=7, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=109), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [8ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [30ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [31ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [31ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [49ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [51ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [52ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [52ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [106ms] refreshing cache from master, [107ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [109ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@299475233(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@29da3de6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1913620629 after retry RPC@1369767084) -> (continuation of Deferred@1615361005 after retry RPC@385856212), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@421c329a -> wakeup thread Time-limited test -> (continuation of Deferred@1913620629 after retry RPC@1369767084) -> (continuation of Deferred@1615361005 after retry RPC@385856212))) after lookup
20:03:49.092 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:49.093 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.094 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.094 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.272 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:49.272 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:49.272 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=9, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=290), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [8ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [30ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [31ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [31ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [49ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [51ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [52ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [52ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [106ms] refreshing cache from master, [107ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [109ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [110ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [112ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [112ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [287ms] refreshing cache from master, [287ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [289ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@299475233(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@29da3de6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1913620629 after retry RPC@1369767084) -> (continuation of Deferred@1615361005 after retry RPC@385856212) -> (continuation of Deferred@1184078068 after retry RPC@1578235850), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@421c329a -> wakeup thread Time-limited test -> (continuation of Deferred@1913620629 after retry RPC@1369767084) -> (continuation of Deferred@1615361005 after retry RPC@385856212) -> (continuation of Deferred@1184078068 after retry RPC@1578235850))) after lookup
20:03:49.274 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:49.275 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.275 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.275 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.791 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:49.792 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:49.792 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=11, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=809), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [8ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [30ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [31ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [31ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [49ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [51ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [52ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [52ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [106ms] refreshing cache from master, [107ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [109ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [110ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [112ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [112ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [287ms] refreshing cache from master, [287ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [289ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [291ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [293ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [293ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [806ms] refreshing cache from master, [806ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [809ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@299475233(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@29da3de6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1913620629 after retry RPC@1369767084) -> (continuation of Deferred@1615361005 after retry RPC@385856212) -> (continuation of Deferred@1184078068 after retry RPC@1578235850) -> (continuation of Deferred@2083738274 after retry RPC@760038729), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@421c329a -> wakeup thread Time-limited test -> (continuation of Deferred@1913620629 after retry RPC@1369767084) -> (continuation of Deferred@1615361005 after retry RPC@385856212) -> (continuation of Deferred@1184078068 after retry RPC@1578235850) -> (continuation of Deferred@2083738274 after retry RPC@760038729))) after lookup
20:03:49.793 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:03:49.794 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.794 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:49.794 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:03:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.291414  6055 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 2fb2124f5c8a4a78b46d0db63c580371)
20:03:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.291551  6055 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.292515  6055 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:50.293517  5744 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:03:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:50.294314  5744 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:03:50.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.297250  5996 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "f37d275110584bbb8c3b91cf61249854" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "533c3ef5bc9342ac84c55607e02629a8" is_pre_election: true
20:03:50.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.297556  5996 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f37d275110584bbb8c3b91cf61249854 in term 3.
20:03:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.298302  5744 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 533c3ef5bc9342ac84c55607e02629a8, f37d275110584bbb8c3b91cf61249854; no voters: 2fb2124f5c8a4a78b46d0db63c580371
20:03:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.298563  6055 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 4 FOLLOWER]: Leader pre-election won for term 5
20:03:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.298622  6055 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 4 FOLLOWER]: Starting leader election (detected failure of leader 2fb2124f5c8a4a78b46d0db63c580371)
20:03:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.298651  6055 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 4 FOLLOWER]: Advancing to term 5
20:03:50.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.299474  6055 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:50.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.299775  6055 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 5 election: Requested vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:50.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.300117  5996 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "f37d275110584bbb8c3b91cf61249854" candidate_term: 5 candidate_status { last_received { term: 4 index: 16 } } ignore_live_leader: false dest_uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:03:50.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.300240  5996 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 3 FOLLOWER]: Advancing to term 5
20:03:50.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:50.300740  5744 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:03:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.301956  5996 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f37d275110584bbb8c3b91cf61249854 in term 5.
20:03:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.302274  5744 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 533c3ef5bc9342ac84c55607e02629a8, f37d275110584bbb8c3b91cf61249854; no voters: 2fb2124f5c8a4a78b46d0db63c580371
20:03:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.302438  6055 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 5 FOLLOWER]: Leader election won for term 5
20:03:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.302649  6055 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 5 LEADER]: Becoming Leader. State: Replica: f37d275110584bbb8c3b91cf61249854, State: Running, Role: LEADER
20:03:50.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.302922  6055 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.305230  4971 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 reported cstate change: term changed from 4 to 5, leader changed from 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66) to f37d275110584bbb8c3b91cf61249854 (127.4.172.67). New cstate: current_term: 5 leader_uuid: "f37d275110584bbb8c3b91cf61249854" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: HEALTHY } } }
20:03:50.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:50.869601  5744 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 -> Peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Couldn't send request to peer 2fb2124f5c8a4a78b46d0db63c580371. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:50.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.909188  5996 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 5 FOLLOWER]: Refusing update from remote peer f37d275110584bbb8c3b91cf61249854: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 5 index: 17. (index mismatch)
20:03:50.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:50.909852  6072 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [LEADER]: Connected to new peer: Peer: permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 17, Last known committed idx: 12, Time since last communication: 0.000s
20:03:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:53.398716  5744 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 -> Peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Couldn't send request to peer 2fb2124f5c8a4a78b46d0db63c580371. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:03:53.851 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:53.852 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:53.852 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=13, sequence_id=13, TimeoutTracker(timeout=30000, elapsed=4869), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] delaying RPC due to: Illegal state: connection is terminated, [6ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [8ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [8ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [26ms] refreshing cache from master, [26ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [28ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [30ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [31ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [31ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [47ms] refreshing cache from master, [47ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [49ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [51ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [52ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [52ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [106ms] refreshing cache from master, [107ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [109ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [110ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [112ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [112ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [287ms] refreshing cache from master, [287ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [289ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [291ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [293ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [293ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [806ms] refreshing cache from master, [806ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [809ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [810ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [812ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [812ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [4866ms] refreshing cache from master, [4866ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [4869ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@299475233(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@29da3de6 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1913620629 after retry RPC@1369767084) -> (continuation of Deferred@1615361005 after retry RPC@385856212) -> (continuation of Deferred@1184078068 after retry RPC@1578235850) -> (continuation of Deferred@2083738274 after retry RPC@760038729) -> (continuation of Deferred@811715072 after retry RPC@2000795208), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@421c329a -> wakeup thread Time-limited test -> (continuation of Deferred@1913620629 after retry RPC@1369767084) -> (continuation of Deferred@1615361005 after retry RPC@385856212) -> (continuation of Deferred@1184078068 after retry RPC@1578235850) -> (continuation of Deferred@2083738274 after retry RPC@760038729) -> (continuation of Deferred@811715072 after retry RPC@2000795208))) after lookup
20:03:53.853 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:53.854 [DEBUG - kudu-nio-0] (Connection.java:757) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] Successfully connected to peer
20:03:53.869 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xcc0b2f7a, L:/127.0.0.1:35548 - R:/127.4.172.67:33409] using TOKEN/null
20:03:53.885 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.4.172.66:44141
20:03:53.885 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:53.886343  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/logs
20:03:53.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:03:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.66:44141
20:03:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.66
20:03:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41487
20:03:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.66
20:03:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:53.994432  6098 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:53.994691  6098 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:53.994737  6098 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:53.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:53.997977  6098 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:53.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:53.998070  6098 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.66
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.001849  6098 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.66:44141
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.66
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41487
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6098
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.66
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/logs
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.003126  6098 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:54.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.004472  6098 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:54.010627  6105 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:54.010767  6106 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.010950  6098 server_base.cc:1061] running on GCE node
20:03:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:54.010703  6108 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.011646  6098 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.012162  6098 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.013334  6098 hybrid_clock.cc:648] HybridClock initialized: now 1779134634013311 us; error 37 us; skew 500 ppm
20:03:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.015162  6098 webserver.cc:533] Webserver started at http://127.4.172.66:41487/ using document root <none> and password file <none>
20:03:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.015750  6098 fs_manager.cc:362] Metadata directory not provided
20:03:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.015810  6098 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:54.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.019268  6098 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
20:03:54.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.021100  6114 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.022081  6098 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.001s
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.022177  6098 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fb2124f5c8a4a78b46d0db63c580371"
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.022619  6098 fs_report.cc:389] FS layout report
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/data
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:54.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:54.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.045892  6098 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:54.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.046566  6098 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:54.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.046710  6098 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:54.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.047209  6098 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:54.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.048410  6121 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
20:03:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.051610  6098 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.051674  6098 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.003s	user 0.000s	sys 0.000s
20:03:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.051738  6098 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:54.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.053279  6098 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:54.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.053321  6098 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
20:03:54.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.053610  6121 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap starting.
20:03:54.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.064381  6098 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.66:44141
20:03:54.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.064395  6228 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.66:44141 every 8 connection(s)
20:03:54.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.065769  6098 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:03:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.069985  6121 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Log is configured to *not* fsync() on all Append() calls
20:03:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.072393  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 6098
20:03:54.072 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:54.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.080613  6229 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:03:54.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.080963  6229 heartbeater.cc:461] Registering TS with master...
20:03:54.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.081645  6229 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:03:54.084 [DEBUG - kudu-nio-0] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 15 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:54.084 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.4.172.67:33409
20:03:54.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.083618  4971 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:54.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.084576  4971 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.66:45901
20:03:54.085 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:54.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.085489  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 5727
20:03:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.087471  6230 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:03:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.087564  6230 heartbeater.cc:461] Registering TS with master...
20:03:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.087703  6230 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:03:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.087940  6231 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:03:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.088008  6231 heartbeater.cc:461] Registering TS with master...
20:03:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.088559  4826 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.088873  6231 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:03:54.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.089972  4897 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:03:54.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.092005  6121 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap replayed 1/1 log segments. Stats: ops{read=16 overwritten=0 applied=16 ignored=0} inserts{seen=12 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:54.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.092370  6121 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap complete.
20:03:54.093 [DEBUG - kudu-nio-0] (Connection.java:239) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] handling channelInactive
20:03:54.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.093384  6121 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Time spent bootstrapping tablet: real 0.040s	user 0.030s	sys 0.000s
20:03:54.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state READY due to: connection closed
20:03:54.094 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:54.095 [DEBUG - Time-limited test] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:54.096 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.096 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.095927  6121 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:54.096 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.096272  6121 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2fb2124f5c8a4a78b46d0db63c580371, State: Initialized, Role: FOLLOWER
20:03:54.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.096778  6121 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 16, Last appended: 4.16, Last appended by leader: 16, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:54.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.097424  6229 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:03:54.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:54.097781  6121 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Time spent starting tablet: real 0.004s	user 0.007s	sys 0.000s
20:03:54.111 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:54.112 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:54.112 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=2, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=17), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [17ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1045520811(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@445863df -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@13a716ab -> wakeup thread Time-limited test)) after lookup
20:03:54.113 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:54.114 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.114 [DEBUG - kudu-nio-2] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.114 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.131 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:54.131 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:54.131 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=4, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=37), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [17ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [20ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [36ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1045520811(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@445863df -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872748242 after retry RPC@9681469), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@13a716ab -> wakeup thread Time-limited test -> (continuation of Deferred@1872748242 after retry RPC@9681469))) after lookup
20:03:54.132 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:54.133 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.133 [DEBUG - kudu-nio-3] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.133 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.151 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:54.152 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:54.152 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=6, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [17ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [20ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [36ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [37ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [39ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [39ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [54ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [57ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1045520811(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@445863df -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872748242 after retry RPC@9681469) -> (continuation of Deferred@1887504532 after retry RPC@1834882795), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@13a716ab -> wakeup thread Time-limited test -> (continuation of Deferred@1872748242 after retry RPC@9681469) -> (continuation of Deferred@1887504532 after retry RPC@1834882795))) after lookup
20:03:54.153 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:54.154 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.154 [DEBUG - kudu-nio-4] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.154 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.292 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:54.292 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:54.292 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=8, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=198), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [17ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [20ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [36ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [37ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [39ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [39ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [54ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [57ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [58ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [60ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [60ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [195ms] refreshing cache from master, [195ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [198ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1045520811(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@445863df -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872748242 after retry RPC@9681469) -> (continuation of Deferred@1887504532 after retry RPC@1834882795) -> (continuation of Deferred@548585970 after retry RPC@1028026945), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@13a716ab -> wakeup thread Time-limited test -> (continuation of Deferred@1872748242 after retry RPC@9681469) -> (continuation of Deferred@1887504532 after retry RPC@1834882795) -> (continuation of Deferred@548585970 after retry RPC@1028026945))) after lookup
20:03:54.293 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:54.294 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.295 [DEBUG - kudu-nio-5] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.295 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.412 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:54.412 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:03:54.412 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=10, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=318), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [17ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [20ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [36ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [37ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [39ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [39ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [54ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [57ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [58ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [60ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [60ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [195ms] refreshing cache from master, [195ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [198ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [199ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [201ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [201ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [315ms] refreshing cache from master, [315ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [318ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1045520811(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@445863df -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872748242 after retry RPC@9681469) -> (continuation of Deferred@1887504532 after retry RPC@1834882795) -> (continuation of Deferred@548585970 after retry RPC@1028026945) -> (continuation of Deferred@1623403801 after retry RPC@1855557434), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@13a716ab -> wakeup thread Time-limited test -> (continuation of Deferred@1872748242 after retry RPC@9681469) -> (continuation of Deferred@1887504532 after retry RPC@1834882795) -> (continuation of Deferred@548585970 after retry RPC@1028026945) -> (continuation of Deferred@1623403801 after retry RPC@1855557434))) after lookup
20:03:54.413 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:03:54.414 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.414 [DEBUG - kudu-nio-6] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:54.415 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:03:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.384989  6242 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 5 FOLLOWER]: Starting pre-election (detected failure of leader f37d275110584bbb8c3b91cf61249854)
20:03:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.385119  6242 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.386118  6242 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:55.387132  5931 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111)
20:03:55.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:55.387918  5931 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409): Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111)
20:03:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.391294  6183 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "533c3ef5bc9342ac84c55607e02629a8" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "2fb2124f5c8a4a78b46d0db63c580371" is_pre_election: true
20:03:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.391605  6183 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 533c3ef5bc9342ac84c55607e02629a8 in term 4.
20:03:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.392387  5930 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, 533c3ef5bc9342ac84c55607e02629a8; no voters: f37d275110584bbb8c3b91cf61249854
20:03:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.392633  6242 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 5 FOLLOWER]: Leader pre-election won for term 6
20:03:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.392709  6242 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 5 FOLLOWER]: Starting leader election (detected failure of leader f37d275110584bbb8c3b91cf61249854)
20:03:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.392752  6242 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 5 FOLLOWER]: Advancing to term 6
20:03:55.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.393573  6242 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:55.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.393975  6242 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 6 election: Requested vote from peers 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:55.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.394398  6183 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "533c3ef5bc9342ac84c55607e02629a8" candidate_term: 6 candidate_status { last_received { term: 5 index: 20 } } ignore_live_leader: false dest_uuid: "2fb2124f5c8a4a78b46d0db63c580371"
20:03:55.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.394570  6183 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 4 FOLLOWER]: Advancing to term 6
20:03:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:55.395041  5931 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409): Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111)
20:03:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.396347  6183 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 533c3ef5bc9342ac84c55607e02629a8 in term 6.
20:03:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.396641  5930 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, 533c3ef5bc9342ac84c55607e02629a8; no voters: f37d275110584bbb8c3b91cf61249854
20:03:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.396797  6242 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 6 FOLLOWER]: Leader election won for term 6
20:03:55.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.397012  6242 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 6 LEADER]: Becoming Leader. State: Replica: 533c3ef5bc9342ac84c55607e02629a8, State: Running, Role: LEADER
20:03:55.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.397294  6242 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 20, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.399750  4971 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 reported cstate change: term changed from 5 to 6, leader changed from f37d275110584bbb8c3b91cf61249854 (127.4.172.67) to 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65). New cstate: current_term: 6 leader_uuid: "533c3ef5bc9342ac84c55607e02629a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: UNKNOWN } } }
20:03:55.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.953486  6183 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 6 FOLLOWER]: Refusing update from remote peer 533c3ef5bc9342ac84c55607e02629a8: Log matching property violated. Preceding OpId in replica: term: 4 index: 16. Preceding OpId from leader: term: 6 index: 21. (index mismatch)
20:03:55.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:55.954172  6260 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 21, Last known committed idx: 16, Time since last communication: 0.000s
20:03:55.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:55.963160  5931 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 -> Peer f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409): Couldn't send request to peer f37d275110584bbb8c3b91cf61249854. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:56.431 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:56.431 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:56.431 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=12, sequence_id=16, TimeoutTracker(timeout=30000, elapsed=2337), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [2ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [14ms] refreshing cache from master, [15ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [17ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [18ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [20ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [34ms] refreshing cache from master, [34ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [36ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [37ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [39ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [39ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [54ms] refreshing cache from master, [55ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [57ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [58ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [60ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [60ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [195ms] refreshing cache from master, [195ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [198ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [199ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [201ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [201ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [315ms] refreshing cache from master, [315ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [318ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [319ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [321ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [321ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2334ms] refreshing cache from master, [2334ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [2336ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1045520811(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@445863df -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1872748242 after retry RPC@9681469) -> (continuation of Deferred@1887504532 after retry RPC@1834882795) -> (continuation of Deferred@548585970 after retry RPC@1028026945) -> (continuation of Deferred@1623403801 after retry RPC@1855557434) -> (continuation of Deferred@291172678 after retry RPC@865623048), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@13a716ab -> wakeup thread Time-limited test -> (continuation of Deferred@1872748242 after retry RPC@9681469) -> (continuation of Deferred@1887504532 after retry RPC@1834882795) -> (continuation of Deferred@548585970 after retry RPC@1028026945) -> (continuation of Deferred@1623403801 after retry RPC@1855557434) -> (continuation of Deferred@291172678 after retry RPC@865623048))) after lookup
20:03:56.432 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:56.433 [DEBUG - kudu-nio-7] (Connection.java:757) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] Successfully connected to peer
20:03:56.460 [DEBUG - kudu-nio-7] (Negotiator.java:919) Authenticated connection [id: 0x5a9c7699, L:/127.0.0.1:40342 - R:/127.4.172.65:46045] using TOKEN/null
20:03:56.489 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.4.172.67:33409
20:03:56.491 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:03:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.491529  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:03:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data
20:03:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/logs
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.67:33409
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.67
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39233
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.67
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:56.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:56.592931  6278 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:56.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:56.593154  6278 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:56.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:56.593216  6278 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:56.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:56.596643  6278 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:56.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:56.596776  6278 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.67
20:03:56.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.600639  6278 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:56.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:56.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.67:33409
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.67
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39233
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6278
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.67
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/logs
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:56.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:03:56.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:03:56.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:03:56.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:03:56.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.602036  6278 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:56.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.603277  6278 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:56.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:56.609359  6284 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:56.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.609431  6278 server_base.cc:1061] running on GCE node
20:03:56.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:56.609360  6283 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:56.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:56.609345  6286 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:56.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.610001  6278 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:56.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.610589  6278 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:56.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.611737  6278 hybrid_clock.cc:648] HybridClock initialized: now 1779134636611715 us; error 33 us; skew 500 ppm
20:03:56.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.613476  6278 webserver.cc:533] Webserver started at http://127.4.172.67:39233/ using document root <none> and password file <none>
20:03:56.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.614034  6278 fs_manager.cc:362] Metadata directory not provided
20:03:56.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.614095  6278 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:56.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.617354  6278 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
20:03:56.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.619189  6292 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:03:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.620101  6278 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
20:03:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.620204  6278 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f37d275110584bbb8c3b91cf61249854"
20:03:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:03:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.620623  6278 fs_report.cc:389] FS layout report
20:03:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:03:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/data
20:03:56.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:56.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:56.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.637418  6278 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:03:56.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.638115  6278 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:56.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.638304  6278 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:56.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.638926  6278 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:56.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.640167  6299 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
20:03:56.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.643680  6278 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:03:56.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.643754  6278 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
20:03:56.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.643815  6278 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:03:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.645772  6278 ts_tablet_manager.cc:616] Registered 1 tablets
20:03:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.645824  6278 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
20:03:56.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.646224  6299 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Bootstrap starting.
20:03:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.656783  6278 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.67:33409
20:03:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.656929  6406 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.67:33409 every 8 connection(s)
20:03:56.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.657918  6278 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:03:56.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.663640  6299 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Log is configured to *not* fsync() on all Append() calls
20:03:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.667186  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 6278
20:03:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.667371  6407 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:03:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.667591  6407 heartbeater.cc:461] Registering TS with master...
20:03:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.667625  6408 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:03:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.667716  6408 heartbeater.cc:461] Registering TS with master...
20:03:56.668 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:56.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.668166  6408 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:03:56.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.668167  6407 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:03:56.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.670265  4826 ts_manager.cc:194] Re-registered known tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.674687  6409 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:03:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.674794  6409 heartbeater.cc:461] Registering TS with master...
20:03:56.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.675004  6409 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:03:56.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.675956  4897 ts_manager.cc:194] Re-registered known tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:56.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.676079  4972 ts_manager.cc:194] Re-registered known tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:56.676 [DEBUG - kudu-nio-7] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 18 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:03:56.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.676970  4972 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.67:41297
20:03:56.677 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.4.172.65:46045
20:03:56.677 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:03:56.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.678056  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 5913
20:03:56.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.682467  6299 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Bootstrap replayed 1/1 log segments. Stats: ops{read=20 overwritten=0 applied=20 ignored=0} inserts{seen=15 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:03:56.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.682875  6299 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Bootstrap complete.
20:03:56.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.684074  6299 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Time spent bootstrapping tablet: real 0.038s	user 0.018s	sys 0.017s
20:03:56.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.686769  6299 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:56.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.687197  6299 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: f37d275110584bbb8c3b91cf61249854, State: Initialized, Role: FOLLOWER
20:03:56.687 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:03:56.687 [DEBUG - kudu-nio-7] (Connection.java:239) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] handling channelInactive
20:03:56.687 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state READY due to: connection closed
20:03:56.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.687767  6299 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 20, Last appended: 5.20, Last appended by leader: 20, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:56.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.688783  6299 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Time spent starting tablet: real 0.005s	user 0.005s	sys 0.001s
20:03:56.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:56.689016  6407 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:03:56.709 [DEBUG - pool-2-thread-1] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:56.711 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.711 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.711 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.731 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:56.731 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:56.732 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=3, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] delaying RPC due to: Illegal state: connection is terminated, [22ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [24ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [24ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [44ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2147378108(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@ef71857 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a7d3725 -> wakeup thread Time-limited test)) after lookup
20:03:56.732 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:56.733 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.733 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.734 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.751 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:56.751 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:56.751 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=5, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=63), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] delaying RPC due to: Illegal state: connection is terminated, [22ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [24ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [24ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [44ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [45ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [46ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [47ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [63ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2147378108(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@ef71857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@439049724 after retry RPC@693933443), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a7d3725 -> wakeup thread Time-limited test -> (continuation of Deferred@439049724 after retry RPC@693933443))) after lookup
20:03:56.752 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:56.753 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.753 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.753 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.811 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:56.812 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:56.812 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=7, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=124), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] delaying RPC due to: Illegal state: connection is terminated, [22ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [24ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [24ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [44ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [45ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [46ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [47ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [63ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [64ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [66ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [66ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [122ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [124ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2147378108(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@ef71857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@439049724 after retry RPC@693933443) -> (continuation of Deferred@941882741 after retry RPC@1242986991), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a7d3725 -> wakeup thread Time-limited test -> (continuation of Deferred@439049724 after retry RPC@693933443) -> (continuation of Deferred@941882741 after retry RPC@1242986991))) after lookup
20:03:56.813 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:56.814 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.814 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:56.814 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:57.071 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:57.071 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:57.071 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=9, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=384), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] delaying RPC due to: Illegal state: connection is terminated, [22ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [24ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [24ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [44ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [45ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [46ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [47ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [63ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [64ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [66ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [66ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [122ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [124ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [126ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [127ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [127ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [381ms] refreshing cache from master, [381ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [383ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2147378108(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@ef71857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@439049724 after retry RPC@693933443) -> (continuation of Deferred@941882741 after retry RPC@1242986991) -> (continuation of Deferred@1259658964 after retry RPC@850661208), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a7d3725 -> wakeup thread Time-limited test -> (continuation of Deferred@439049724 after retry RPC@693933443) -> (continuation of Deferred@941882741 after retry RPC@1242986991) -> (continuation of Deferred@1259658964 after retry RPC@850661208))) after lookup
20:03:57.072 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:57.073 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:57.073 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:57.074 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:57.191 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:03:57.191 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:03:57.192 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=11, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=504), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] delaying RPC due to: Illegal state: connection is terminated, [22ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [24ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [24ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [44ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [45ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [46ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [47ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [63ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [64ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [66ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [66ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [122ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [124ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [126ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [127ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [127ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [381ms] refreshing cache from master, [381ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [383ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [385ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [387ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [387ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [501ms] refreshing cache from master, [501ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [504ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2147378108(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@ef71857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@439049724 after retry RPC@693933443) -> (continuation of Deferred@941882741 after retry RPC@1242986991) -> (continuation of Deferred@1259658964 after retry RPC@850661208) -> (continuation of Deferred@826300787 after retry RPC@589061836), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a7d3725 -> wakeup thread Time-limited test -> (continuation of Deferred@439049724 after retry RPC@693933443) -> (continuation of Deferred@941882741 after retry RPC@1242986991) -> (continuation of Deferred@1259658964 after retry RPC@850661208) -> (continuation of Deferred@826300787 after retry RPC@589061836))) after lookup
20:03:57.192 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:03:57.193 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:57.193 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:57.193 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:03:57.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.989167  6420 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 6 FOLLOWER]: Starting pre-election (detected failure of leader 533c3ef5bc9342ac84c55607e02629a8)
20:03:57.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.989311  6420 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 6 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:57.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.990307  6420 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 7 pre-election: Requested pre-vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:57.990971  6117 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111)
20:03:57.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:57.991832  6117 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 7 pre-election: RPC error from VoteRequest() call to peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111)
20:03:57.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.995241  6361 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "2fb2124f5c8a4a78b46d0db63c580371" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "f37d275110584bbb8c3b91cf61249854" is_pre_election: true
20:03:57.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.995509  6361 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2fb2124f5c8a4a78b46d0db63c580371 in term 5.
20:03:57.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.996177  6118 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 7 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, f37d275110584bbb8c3b91cf61249854; no voters: 533c3ef5bc9342ac84c55607e02629a8
20:03:57.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.996428  6420 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 6 FOLLOWER]: Leader pre-election won for term 7
20:03:57.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.996493  6420 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 6 FOLLOWER]: Starting leader election (detected failure of leader 533c3ef5bc9342ac84c55607e02629a8)
20:03:57.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.996534  6420 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 6 FOLLOWER]: Advancing to term 7
20:03:57.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.997341  6420 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 7 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:57.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.997727  6420 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 7 election: Requested vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:03:57.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.998122  6361 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "2fb2124f5c8a4a78b46d0db63c580371" candidate_term: 7 candidate_status { last_received { term: 6 index: 24 } } ignore_live_leader: false dest_uuid: "f37d275110584bbb8c3b91cf61249854"
20:03:57.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:57.998278  6361 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 5 FOLLOWER]: Advancing to term 7
20:03:57.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:57.998854  6117 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 7 election: RPC error from VoteRequest() call to peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111)
20:03:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:58.000288  6361 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 7 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2fb2124f5c8a4a78b46d0db63c580371 in term 7.
20:03:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:58.000581  6118 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 7 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, f37d275110584bbb8c3b91cf61249854; no voters: 533c3ef5bc9342ac84c55607e02629a8
20:03:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:58.000746  6420 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 7 FOLLOWER]: Leader election won for term 7
20:03:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:58.000969  6420 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 7 LEADER]: Becoming Leader. State: Replica: 2fb2124f5c8a4a78b46d0db63c580371, State: Running, Role: LEADER
20:03:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:58.001302  6420 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 7, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:03:58.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:58.003664  4972 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 reported cstate change: term changed from 6 to 7, leader changed from 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65) to 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66). New cstate: current_term: 7 leader_uuid: "2fb2124f5c8a4a78b46d0db63c580371" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: UNKNOWN } } }
20:03:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:03:58.572777  6117 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 -> Peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Couldn't send request to peer 533c3ef5bc9342ac84c55607e02629a8. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:03:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:58.624290  6361 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 7 FOLLOWER]: Refusing update from remote peer 2fb2124f5c8a4a78b46d0db63c580371: Log matching property violated. Preceding OpId in replica: term: 5 index: 20. Preceding OpId from leader: term: 7 index: 25. (index mismatch)
20:03:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:03:58.625020  6437 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 20, Time since last communication: 0.000s
20:04:01.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:01.008066  6117 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 -> Peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Couldn't send request to peer 533c3ef5bc9342ac84c55607e02629a8. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:04:01.272 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:01.272 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:01.272 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=13, sequence_id=19, TimeoutTracker(timeout=30000, elapsed=4584), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] delaying RPC due to: Illegal state: connection is terminated, [22ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [24ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [24ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [42ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [44ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [45ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [46ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [47ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [63ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [64ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [66ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [66ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [122ms] refreshing cache from master, [122ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [124ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [126ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [127ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [127ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [381ms] refreshing cache from master, [381ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [383ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [385ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [387ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [387ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [501ms] refreshing cache from master, [501ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [504ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [505ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [506ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [506ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [4582ms] refreshing cache from master, [4582ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [4584ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2147378108(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@ef71857 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@439049724 after retry RPC@693933443) -> (continuation of Deferred@941882741 after retry RPC@1242986991) -> (continuation of Deferred@1259658964 after retry RPC@850661208) -> (continuation of Deferred@826300787 after retry RPC@589061836) -> (continuation of Deferred@1447116022 after retry RPC@401269452), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5a7d3725 -> wakeup thread Time-limited test -> (continuation of Deferred@439049724 after retry RPC@693933443) -> (continuation of Deferred@941882741 after retry RPC@1242986991) -> (continuation of Deferred@1259658964 after retry RPC@850661208) -> (continuation of Deferred@826300787 after retry RPC@589061836) -> (continuation of Deferred@1447116022 after retry RPC@401269452))) after lookup
20:04:01.273 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:01.274 [DEBUG - kudu-nio-6] (Connection.java:757) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] Successfully connected to peer
20:04:01.302 [DEBUG - kudu-nio-6] (Negotiator.java:919) Authenticated connection [id: 0x8259c527, L:/127.0.0.1:40488 - R:/127.4.172.66:44141] using TOKEN/null
20:04:01.321 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.4.172.65:46045
20:04:01.322 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:04:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.322799  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/logs
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.65:46045
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.65
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43765
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.65
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:01.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:01.424944  6463 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:01.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:01.425163  6463 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:01.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:01.425209  6463 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:01.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:01.428673  6463 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:01.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:01.428802  6463 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.65
20:04:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.432623  6463 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:04:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.65:46045
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.65
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43765
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6463
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.65
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/logs
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:04:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:04:01.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:04:01.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.433799  6463 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:01.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.434944  6463 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:01.441218  6468 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:01.441257  6469 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.441401  6463 server_base.cc:1061] running on GCE node
20:04:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:01.441218  6471 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.442090  6463 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.442629  6463 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.443773  6463 hybrid_clock.cc:648] HybridClock initialized: now 1779134641443755 us; error 35 us; skew 500 ppm
20:04:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.445502  6463 webserver.cc:533] Webserver started at http://127.4.172.65:43765/ using document root <none> and password file <none>
20:04:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.446028  6463 fs_manager.cc:362] Metadata directory not provided
20:04:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.446080  6463 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.449474  6463 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.000s
20:04:01.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.451833  6477 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.457763  6463 fs_manager.cc:730] Time spent opening block manager: real 0.007s	user 0.003s	sys 0.000s
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.458006  6463 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.458494  6463 fs_report.cc:389] FS layout report
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/data
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:01.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.479249  6463 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:01.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.479897  6463 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:01.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.480048  6463 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:01.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.480585  6463 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:01.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.481815  6484 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
20:04:01.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.485116  6463 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:01.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.485181  6463 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
20:04:01.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.485252  6463 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:01.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.486750  6463 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:01.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.486796  6463 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
20:04:01.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.487138  6484 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Bootstrap starting.
20:04:01.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.496596  6463 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.65:46045
20:04:01.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.496652  6591 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.65:46045 every 8 connection(s)
20:04:01.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.497673  6463 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:04:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.498756  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 6463
20:04:01.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.506145  6484 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Log is configured to *not* fsync() on all Append() calls
20:04:01.508 [DEBUG - kudu-nio-6] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 21 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:01.509 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.4.172.66:44141
20:04:01.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:04:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.510130  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 6098
20:04:01.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.514714  6593 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:04:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.514961  6593 heartbeater.cc:461] Registering TS with master...
20:04:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.515522  6593 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:04:01.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.517410  4826 ts_manager.cc:194] Re-registered known tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:04:01.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.517721  6592 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:04:01.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.517805  6592 heartbeater.cc:461] Registering TS with master...
20:04:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.518013  6592 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:04:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.518812  4971 ts_manager.cc:194] Re-registered known tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:04:01.519 [DEBUG - kudu-nio-6] (Connection.java:239) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] handling channelInactive
20:04:01.519 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state READY due to: connection closed
20:04:01.519 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.519716  4971 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.65:41165
20:04:01.520 [DEBUG - Time-limited test] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.520337  6594 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:04:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.520432  6594 heartbeater.cc:461] Registering TS with master...
20:04:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.520612  6594 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:04:01.521 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.521 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.521 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.521597  4900 ts_manager.cc:194] Re-registered known tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:04:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.525573  6484 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Bootstrap replayed 1/1 log segments. Stats: ops{read=24 overwritten=0 applied=24 ignored=0} inserts{seen=18 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.525866  6484 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Bootstrap complete.
20:04:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.526681  6484 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Time spent bootstrapping tablet: real 0.040s	user 0.020s	sys 0.016s
20:04:01.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.528573  6484 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 6 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:01.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.528882  6484 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 6 FOLLOWER]: Becoming Follower/Learner. State: Replica: 533c3ef5bc9342ac84c55607e02629a8, State: Initialized, Role: FOLLOWER
20:04:01.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.529330  6484 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 24, Last appended: 6.24, Last appended by leader: 24, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:01.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.530001  6592 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:04:01.532 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:01.530270  6484 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Time spent starting tablet: real 0.004s	user 0.002s	sys 0.001s
20:04:01.532 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:01.532 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=2, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=12), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [1ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [11ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@361657346(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@553208fd -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@fa085e5 -> wakeup thread Time-limited test)) after lookup
20:04:01.533 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:01.534 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.534 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.534 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.551 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:01.551 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:01.551 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=4, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=31), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [1ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [11ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [13ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [14ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [31ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@361657346(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@553208fd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550220914 after retry RPC@1474451382), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@fa085e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1550220914 after retry RPC@1474451382))) after lookup
20:04:01.552 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:01.553 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.553 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.553 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.591 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:01.591 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:01.591 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=6, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=71), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [1ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [11ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [13ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [14ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [31ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [32ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [33ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [33ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [70ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@361657346(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@553208fd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550220914 after retry RPC@1474451382) -> (continuation of Deferred@483932793 after retry RPC@1505919411), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@fa085e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1550220914 after retry RPC@1474451382) -> (continuation of Deferred@483932793 after retry RPC@1505919411))) after lookup
20:04:01.592 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:01.593 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.593 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.593 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.671 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:01.671 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:01.671 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=8, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=151), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [1ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [11ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [13ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [14ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [31ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [32ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [33ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [33ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [70ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [72ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [73ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [73ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [148ms] refreshing cache from master, [148ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [150ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@361657346(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@553208fd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550220914 after retry RPC@1474451382) -> (continuation of Deferred@483932793 after retry RPC@1505919411) -> (continuation of Deferred@1153862011 after retry RPC@746701891), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@fa085e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1550220914 after retry RPC@1474451382) -> (continuation of Deferred@483932793 after retry RPC@1505919411) -> (continuation of Deferred@1153862011 after retry RPC@746701891))) after lookup
20:04:01.672 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:01.673 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.673 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.673 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.831 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:01.832 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:01.832 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=10, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=312), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [1ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [11ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [13ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [14ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [31ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [32ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [33ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [33ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [70ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [72ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [73ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [73ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [148ms] refreshing cache from master, [148ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [150ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [151ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [153ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [153ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [309ms] refreshing cache from master, [309ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [311ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@361657346(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@553208fd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550220914 after retry RPC@1474451382) -> (continuation of Deferred@483932793 after retry RPC@1505919411) -> (continuation of Deferred@1153862011 after retry RPC@746701891) -> (continuation of Deferred@132671575 after retry RPC@636032821), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@fa085e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1550220914 after retry RPC@1474451382) -> (continuation of Deferred@483932793 after retry RPC@1505919411) -> (continuation of Deferred@1153862011 after retry RPC@746701891) -> (continuation of Deferred@132671575 after retry RPC@636032821))) after lookup
20:04:01.833 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:01.834 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.834 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:01.834 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.821904  6605 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 7 FOLLOWER]: Starting pre-election (detected failure of leader 2fb2124f5c8a4a78b46d0db63c580371)
20:04:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.822059  6605 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 7 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.823086  6605 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 8 pre-election: Requested pre-vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:04:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:02.823956  6295 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:04:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:02.824769  6295 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 8 pre-election: RPC error from VoteRequest() call to peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.827912  6546 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "f37d275110584bbb8c3b91cf61249854" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "533c3ef5bc9342ac84c55607e02629a8" is_pre_election: true
20:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.828164  6546 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 6 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f37d275110584bbb8c3b91cf61249854 in term 6.
20:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.828790  6295 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 8 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 533c3ef5bc9342ac84c55607e02629a8, f37d275110584bbb8c3b91cf61249854; no voters: 2fb2124f5c8a4a78b46d0db63c580371
20:04:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.829002  6605 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 7 FOLLOWER]: Leader pre-election won for term 8
20:04:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.829070  6605 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 7 FOLLOWER]: Starting leader election (detected failure of leader 2fb2124f5c8a4a78b46d0db63c580371)
20:04:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.829113  6605 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 7 FOLLOWER]: Advancing to term 8
20:04:02.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.829969  6605 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 8 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:02.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.830329  6605 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 8 election: Requested vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:04:02.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.830716  6546 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "f37d275110584bbb8c3b91cf61249854" candidate_term: 8 candidate_status { last_received { term: 7 index: 28 } } ignore_live_leader: false dest_uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:04:02.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.830833  6546 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 6 FOLLOWER]: Advancing to term 8
20:04:02.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:02.831303  6295 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 8 election: RPC error from VoteRequest() call to peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:04:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.832530  6546 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 8 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f37d275110584bbb8c3b91cf61249854 in term 8.
20:04:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.832829  6295 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 8 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 533c3ef5bc9342ac84c55607e02629a8, f37d275110584bbb8c3b91cf61249854; no voters: 2fb2124f5c8a4a78b46d0db63c580371
20:04:02.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.833001  6605 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 8 FOLLOWER]: Leader election won for term 8
20:04:02.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.833184  6605 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 8 LEADER]: Becoming Leader. State: Replica: f37d275110584bbb8c3b91cf61249854, State: Running, Role: LEADER
20:04:02.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.833446  6605 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 8, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:02.835695  4971 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 reported cstate change: term changed from 7 to 8, leader changed from 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66) to f37d275110584bbb8c3b91cf61249854 (127.4.172.67). New cstate: current_term: 8 leader_uuid: "f37d275110584bbb8c3b91cf61249854" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: HEALTHY } } }
20:04:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:03.400974  6295 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 -> Peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Couldn't send request to peer 2fb2124f5c8a4a78b46d0db63c580371. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.425511  6546 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 8 FOLLOWER]: Refusing update from remote peer f37d275110584bbb8c3b91cf61249854: Log matching property violated. Preceding OpId in replica: term: 6 index: 24. Preceding OpId from leader: term: 8 index: 29. (index mismatch)
20:04:03.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.426231  6622 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [LEADER]: Connected to new peer: Peer: permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 24, Time since last communication: 0.000s
20:04:03.432 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:03.432 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:04:03.433 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=12, sequence_id=22, TimeoutTracker(timeout=30000, elapsed=1913), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [1ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [1ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [9ms] refreshing cache from master, [9ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [11ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [13ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [14ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [14ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [28ms] refreshing cache from master, [28ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [31ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [32ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [33ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [33ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [68ms] refreshing cache from master, [68ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [70ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [72ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [73ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [73ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [148ms] refreshing cache from master, [148ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [150ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [151ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [153ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [153ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [309ms] refreshing cache from master, [309ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [311ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [312ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [314ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [314ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [1909ms] refreshing cache from master, [1909ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [1912ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@361657346(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@553208fd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1550220914 after retry RPC@1474451382) -> (continuation of Deferred@483932793 after retry RPC@1505919411) -> (continuation of Deferred@1153862011 after retry RPC@746701891) -> (continuation of Deferred@132671575 after retry RPC@636032821) -> (continuation of Deferred@168485053 after retry RPC@1740054134), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@fa085e5 -> wakeup thread Time-limited test -> (continuation of Deferred@1550220914 after retry RPC@1474451382) -> (continuation of Deferred@483932793 after retry RPC@1505919411) -> (continuation of Deferred@1153862011 after retry RPC@746701891) -> (continuation of Deferred@132671575 after retry RPC@636032821) -> (continuation of Deferred@168485053 after retry RPC@1740054134))) after lookup
20:04:03.433 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:04:03.435 [DEBUG - kudu-nio-5] (Connection.java:757) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] Successfully connected to peer
20:04:03.472 [DEBUG - kudu-nio-5] (Negotiator.java:919) Authenticated connection [id: 0x684ce8de, L:/127.0.0.1:41964 - R:/127.4.172.67:33409] using TOKEN/null
20:04:03.496 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.4.172.66:44141
20:04:03.496 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.497125  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/logs
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:04:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.66:44141
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.66
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41487
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.66
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:03.602910  6636 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:03.603112  6636 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:03.603156  6636 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:03.606383  6636 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:03.606488  6636 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.66
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.610214  6636 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.66:44141
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.66
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41487
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6636
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.66
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/logs
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:04:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.611342  6636 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.612437  6636 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:03.618736  6644 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:03.618742  6641 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:03.618749  6642 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.618968  6636 server_base.cc:1061] running on GCE node
20:04:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.619299  6636 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.619818  6636 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:03.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.620970  6636 hybrid_clock.cc:648] HybridClock initialized: now 1779134643620958 us; error 28 us; skew 500 ppm
20:04:03.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.622782  6636 webserver.cc:533] Webserver started at http://127.4.172.66:41487/ using document root <none> and password file <none>
20:04:03.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.623314  6636 fs_manager.cc:362] Metadata directory not provided
20:04:03.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.623374  6636 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:03.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.626601  6636 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.004s	sys 0.000s
20:04:03.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.628285  6650 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.629168  6636 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
20:04:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.629266  6636 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:04:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fb2124f5c8a4a78b46d0db63c580371"
20:04:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:04:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.629664  6636 fs_report.cc:389] FS layout report
20:04:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:04:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/data
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.658365  6636 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:03.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.659256  6636 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:03.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.659437  6636 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:03.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.660084  6636 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:03.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.661643  6657 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
20:04:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.665158  6636 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.665222  6636 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.001s	sys 0.000s
20:04:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.665282  6636 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:03.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.667081  6636 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:03.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.667135  6636 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
20:04:03.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.667487  6657 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap starting.
20:04:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.677203  6636 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.66:44141
20:04:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.677239  6764 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.66:44141 every 8 connection(s)
20:04:03.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.678205  6636 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:04:03.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.683841  6657 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Log is configured to *not* fsync() on all Append() calls
20:04:03.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.685485  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 6636
20:04:03.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.688758  6765 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:04:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.688997  6765 heartbeater.cc:461] Registering TS with master...
20:04:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.689116  6766 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:04:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.689178  6766 heartbeater.cc:461] Registering TS with master...
20:04:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.689635  6765 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:04:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.689635  6766 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:04:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.691527  4826 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.692152  4972 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:04:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.693073  4972 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.66:59181
20:04:03.694 [DEBUG - kudu-nio-5] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 24 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:03.696 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.4.172.67:33409
20:04:03.696 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:04:03.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.698210  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 6278
20:04:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.702724  6767 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:04:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.702836  6767 heartbeater.cc:461] Registering TS with master...
20:04:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.703015  6767 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:04:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.704128  4900 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:04:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.704756  6657 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=21 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:03.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.705152  6657 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap complete.
20:04:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.706372  6657 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Time spent bootstrapping tablet: real 0.039s	user 0.034s	sys 0.000s
20:04:03.707 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:03.707 [DEBUG - kudu-nio-5] (Connection.java:239) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] handling channelInactive
20:04:03.707 [DEBUG - kudu-nio-5] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state READY due to: connection closed
20:04:03.708 [DEBUG - Time-limited test] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:04:03.709 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.709 [DEBUG - kudu-nio-6] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.709057  6657 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 7 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:03.709 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.709489  6657 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 7 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2fb2124f5c8a4a78b46d0db63c580371, State: Initialized, Role: FOLLOWER
20:04:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.710024  6657 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 7.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.710687  6765 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:04:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:03.711032  6657 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Time spent starting tablet: real 0.005s	user 0.008s	sys 0.000s
20:04:03.731 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:03.731 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:04:03.731 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=2, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=24), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [1ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [1ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@946998324(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@67e9e1e2 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@62295667 -> wakeup thread Time-limited test)) after lookup
20:04:03.732 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:04:03.733 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.733 [DEBUG - kudu-nio-7] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.733 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.751 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:03.751 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:04:03.751 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=4, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [1ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [1ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [25ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [25ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [42ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@946998324(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@67e9e1e2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@119554356 after retry RPC@798139865), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@62295667 -> wakeup thread Time-limited test -> (continuation of Deferred@119554356 after retry RPC@798139865))) after lookup
20:04:03.752 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:04:03.753 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.753 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.771 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:03.771 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:04:03.772 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=6, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=64), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [1ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [1ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [25ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [25ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [42ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [43ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [45ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [45ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [63ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@946998324(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@67e9e1e2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@119554356 after retry RPC@798139865) -> (continuation of Deferred@1602948246 after retry RPC@1695405779), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@62295667 -> wakeup thread Time-limited test -> (continuation of Deferred@119554356 after retry RPC@798139865) -> (continuation of Deferred@1602948246 after retry RPC@1695405779))) after lookup
20:04:03.772 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:04:03.773 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.774 [DEBUG - kudu-nio-1] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.774 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.851 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:03.851 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:04:03.852 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=8, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=144), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [1ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [1ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [25ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [25ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [42ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [43ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [45ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [45ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [63ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [64ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [66ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [66ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [140ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [143ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@946998324(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@67e9e1e2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@119554356 after retry RPC@798139865) -> (continuation of Deferred@1602948246 after retry RPC@1695405779) -> (continuation of Deferred@1348666052 after retry RPC@1540619605), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@62295667 -> wakeup thread Time-limited test -> (continuation of Deferred@119554356 after retry RPC@798139865) -> (continuation of Deferred@1602948246 after retry RPC@1695405779) -> (continuation of Deferred@1348666052 after retry RPC@1540619605))) after lookup
20:04:03.852 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:04:03.853 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.853 [DEBUG - kudu-nio-2] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:03.853 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:04.331 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:04.331 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:04:04.331 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=10, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=623), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [1ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [1ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [25ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [25ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [42ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [43ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [45ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [45ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [63ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [64ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [66ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [66ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [140ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [143ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [144ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [145ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [145ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [620ms] refreshing cache from master, [620ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [622ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@946998324(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@67e9e1e2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@119554356 after retry RPC@798139865) -> (continuation of Deferred@1602948246 after retry RPC@1695405779) -> (continuation of Deferred@1348666052 after retry RPC@1540619605) -> (continuation of Deferred@2142718305 after retry RPC@271277861), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@62295667 -> wakeup thread Time-limited test -> (continuation of Deferred@119554356 after retry RPC@798139865) -> (continuation of Deferred@1602948246 after retry RPC@1695405779) -> (continuation of Deferred@1348666052 after retry RPC@1540619605) -> (continuation of Deferred@2142718305 after retry RPC@271277861))) after lookup
20:04:04.332 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:04:04.333 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:04.333 [DEBUG - kudu-nio-3] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state CONNECTING due to: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:04.333 [INFO - kudu-nio-3] (AsyncKuduClient.java:2355) Invalidating location f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409
20:04:04.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:04.994828  6778 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 8 FOLLOWER]: Starting pre-election (detected failure of leader f37d275110584bbb8c3b91cf61249854)
20:04:04.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:04.994984  6778 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 8 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:04.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:04.996136  6778 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 9 pre-election: Requested pre-vote from peers 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:04:04.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:04.996845  6481 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111)
20:04:04.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:04.997710  6481 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 9 pre-election: RPC error from VoteRequest() call to peer f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409): Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111)
20:04:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.000797  6719 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "533c3ef5bc9342ac84c55607e02629a8" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "2fb2124f5c8a4a78b46d0db63c580371" is_pre_election: true
20:04:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.001055  6719 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 7 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 533c3ef5bc9342ac84c55607e02629a8 in term 7.
20:04:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.001740  6480 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 9 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, 533c3ef5bc9342ac84c55607e02629a8; no voters: f37d275110584bbb8c3b91cf61249854
20:04:05.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.001971  6778 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 8 FOLLOWER]: Leader pre-election won for term 9
20:04:05.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.002035  6778 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 8 FOLLOWER]: Starting leader election (detected failure of leader f37d275110584bbb8c3b91cf61249854)
20:04:05.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.002063  6778 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 8 FOLLOWER]: Advancing to term 9
20:04:05.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.002861  6778 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 9 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.003175  6778 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 9 election: Requested vote from peers 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:04:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.003562  6719 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "533c3ef5bc9342ac84c55607e02629a8" candidate_term: 9 candidate_status { last_received { term: 8 index: 32 } } ignore_live_leader: false dest_uuid: "2fb2124f5c8a4a78b46d0db63c580371"
20:04:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.003700  6719 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 7 FOLLOWER]: Advancing to term 9
20:04:05.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:05.004150  6481 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 9 election: RPC error from VoteRequest() call to peer f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409): Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111)
20:04:05.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.005362  6719 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 9 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 533c3ef5bc9342ac84c55607e02629a8 in term 9.
20:04:05.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.005658  6480 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [CANDIDATE]: Term 9 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, 533c3ef5bc9342ac84c55607e02629a8; no voters: f37d275110584bbb8c3b91cf61249854
20:04:05.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.005811  6778 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 9 FOLLOWER]: Leader election won for term 9
20:04:05.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.005991  6778 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 9 LEADER]: Becoming Leader. State: Replica: 533c3ef5bc9342ac84c55607e02629a8, State: Running, Role: LEADER
20:04:05.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.006301  6778 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 32, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 9, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:05.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.008708  4972 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 reported cstate change: term changed from 8 to 9, leader changed from f37d275110584bbb8c3b91cf61249854 (127.4.172.67) to 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65). New cstate: current_term: 9 leader_uuid: "533c3ef5bc9342ac84c55607e02629a8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: UNKNOWN } } }
20:04:05.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.558404  6719 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 9 FOLLOWER]: Refusing update from remote peer 533c3ef5bc9342ac84c55607e02629a8: Log matching property violated. Preceding OpId in replica: term: 7 index: 28. Preceding OpId from leader: term: 9 index: 33. (index mismatch)
20:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.559090  6797 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 33, Last known committed idx: 28, Time since last communication: 0.000s
20:04:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:05.563990  6799 mvcc.cc:204] Tried to move back new op lower bound from 7287335499674165248 to 7287335497048313856. Current Snapshot: MvccSnapshot[applied={T|T < 7287335499674165248}]
20:04:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:05.574378  6481 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 -> Peer f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409): Couldn't send request to peer f37d275110584bbb8c3b91cf61249854. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.67:33409: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:06.190 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:06.191 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:06.191 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=12, sequence_id=25, TimeoutTracker(timeout=30000, elapsed=2483), Traces: [0ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [1ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [1ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [20ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [25ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [25ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [40ms] refreshing cache from master, [40ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [42ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [43ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [45ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [45ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [61ms] refreshing cache from master, [61ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [63ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [64ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [66ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [66ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [140ms] refreshing cache from master, [141ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [143ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [144ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [145ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [145ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [620ms] refreshing cache from master, [620ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [622ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [624ms] sending RPC to server f37d275110584bbb8c3b91cf61249854, [625ms] received response from server f37d275110584bbb8c3b91cf61249854: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [625ms] delaying RPC due to: Network error: Failed to connect to peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409): Connection refused: /127.4.172.67:33409, [2480ms] refreshing cache from master, [2480ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [2482ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@946998324(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@67e9e1e2 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@119554356 after retry RPC@798139865) -> (continuation of Deferred@1602948246 after retry RPC@1695405779) -> (continuation of Deferred@1348666052 after retry RPC@1540619605) -> (continuation of Deferred@2142718305 after retry RPC@271277861) -> (continuation of Deferred@2056175210 after retry RPC@806749289), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@62295667 -> wakeup thread Time-limited test -> (continuation of Deferred@119554356 after retry RPC@798139865) -> (continuation of Deferred@1602948246 after retry RPC@1695405779) -> (continuation of Deferred@1348666052 after retry RPC@1540619605) -> (continuation of Deferred@2142718305 after retry RPC@271277861) -> (continuation of Deferred@2056175210 after retry RPC@806749289))) after lookup
20:04:06.192 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:04:06.193 [DEBUG - kudu-nio-4] (Connection.java:757) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] Successfully connected to peer
20:04:06.221 [DEBUG - kudu-nio-4] (Negotiator.java:919) Authenticated connection [id: 0x9c0a8798, L:/127.0.0.1:59096 - R:/127.4.172.65:46045] using TOKEN/null
20:04:06.244 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.4.172.67:33409
20:04:06.244 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 2
  }
}

20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.245026  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/logs
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.67:33409
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.67
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39233
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.67
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:06.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:06.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:06.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:06.343609  6817 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:06.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:06.343845  6817 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:06.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:06.343904  6817 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:06.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:06.347250  6817 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:06.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:06.347368  6817 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.67
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.351153  6817 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.67:33409
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:04:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.67
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39233
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6817
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.67
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/logs
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:04:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.352547  6817 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:06.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.353689  6817 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:06.360015  6822 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:06.359913  6823 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:06.359930  6825 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.360852  6817 server_base.cc:1061] running on GCE node
20:04:06.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.361218  6817 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:06.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.361763  6817 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:06.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.362934  6817 hybrid_clock.cc:648] HybridClock initialized: now 1779134646362914 us; error 36 us; skew 500 ppm
20:04:06.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.364672  6817 webserver.cc:533] Webserver started at http://127.4.172.67:39233/ using document root <none> and password file <none>
20:04:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.365223  6817 fs_manager.cc:362] Metadata directory not provided
20:04:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.365284  6817 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:06.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.368589  6817 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
20:04:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.370441  6831 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.371444  6817 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.000s
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.371557  6817 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f37d275110584bbb8c3b91cf61249854"
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.372022  6817 fs_report.cc:389] FS layout report
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/wal
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/data
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:06.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.399451  6817 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.400188  6817 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.400372  6817 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:06.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.400995  6817 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:06.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.402366  6838 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
20:04:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.405709  6817 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.405768  6817 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
20:04:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.405829  6817 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.407341  6817 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.407397  6817 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.001s	sys 0.000s
20:04:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.407722  6838 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Bootstrap starting.
20:04:06.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.418505  6817 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.67:33409
20:04:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.418931  6945 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.67:33409 every 8 connection(s)
20:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.420217  6817 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-2/data/info.pb
20:04:06.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.427227  6838 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Log is configured to *not* fsync() on all Append() calls
20:04:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.430413  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 6817
20:04:06.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.434504  6947 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:04:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.434734  6947 heartbeater.cc:461] Registering TS with master...
20:04:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.435348  6947 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:04:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.436288  6949 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:04:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.436367  6949 heartbeater.cc:461] Registering TS with master...
20:04:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.436543  6949 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:04:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.437109  4972 ts_manager.cc:194] Re-registered known tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:04:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.437992  4972 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.67:42079
20:04:06.439 [DEBUG - kudu-nio-4] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 27 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.438482  6948 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:04:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.438580  6948 heartbeater.cc:461] Registering TS with master...
20:04:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.438750  6948 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:04:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.439592  4826 ts_manager.cc:194] Re-registered known tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:04:06.440 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.4.172.65:46045
20:04:06.440 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:04:06.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.440642  4900 ts_manager.cc:194] Re-registered known tserver with Master: f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:04:06.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.440996  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 6463
20:04:06.447 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:06.447 [DEBUG - kudu-nio-4] (Connection.java:239) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] handling channelInactive
20:04:06.447 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state READY due to: connection closed
20:04:06.448 [DEBUG - Time-limited test] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:04:06.449 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.449 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.449 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.460714  6838 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Bootstrap replayed 1/1 log segments. Stats: ops{read=32 overwritten=0 applied=32 ignored=0} inserts{seen=24 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.461127  6838 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Bootstrap complete.
20:04:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.462222  6838 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Time spent bootstrapping tablet: real 0.055s	user 0.029s	sys 0.008s
20:04:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.464222  6838 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 8 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.464565  6838 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 8 FOLLOWER]: Becoming Follower/Learner. State: Replica: f37d275110584bbb8c3b91cf61249854, State: Initialized, Role: FOLLOWER
20:04:06.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.464974  6838 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 32, Last appended: 8.32, Last appended by leader: 32, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:06.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.465878  6838 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854: Time spent starting tablet: real 0.004s	user 0.007s	sys 0.000s
20:04:06.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:06.466027  6947 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:04:06.471 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:06.471 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:06.471 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=2, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [22ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1846377997(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6e381ae9 -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c39a70e -> wakeup thread Time-limited test)) after lookup
20:04:06.472 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:04:06.473 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.473 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.473 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.491 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:06.492 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:06.492 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=4, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=44), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [22ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [23ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [25ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1846377997(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6e381ae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1777092961 after retry RPC@2021429893), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c39a70e -> wakeup thread Time-limited test -> (continuation of Deferred@1777092961 after retry RPC@2021429893))) after lookup
20:04:06.492 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:04:06.494 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.494 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.494 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.531 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:06.531 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:06.532 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=6, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=84), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [22ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [23ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [25ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [44ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [46ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [46ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [83ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1846377997(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6e381ae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1777092961 after retry RPC@2021429893) -> (continuation of Deferred@829487004 after retry RPC@763671437), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c39a70e -> wakeup thread Time-limited test -> (continuation of Deferred@1777092961 after retry RPC@2021429893) -> (continuation of Deferred@829487004 after retry RPC@763671437))) after lookup
20:04:06.532 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:04:06.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.533 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.570 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:06.571 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:06.571 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=8, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=123), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [22ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [23ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [25ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [44ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [46ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [46ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [83ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [84ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [85ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [85ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [122ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1846377997(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6e381ae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1777092961 after retry RPC@2021429893) -> (continuation of Deferred@829487004 after retry RPC@763671437) -> (continuation of Deferred@357006246 after retry RPC@500673112), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c39a70e -> wakeup thread Time-limited test -> (continuation of Deferred@1777092961 after retry RPC@2021429893) -> (continuation of Deferred@829487004 after retry RPC@763671437) -> (continuation of Deferred@357006246 after retry RPC@500673112))) after lookup
20:04:06.572 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:04:06.573 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.573 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.573 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.831 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:06.831 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)[L],f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:06.831 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=10, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=384), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [22ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [23ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [25ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [44ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [46ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [46ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [83ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [84ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [85ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [85ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [122ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [123ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [125ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [125ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [380ms] refreshing cache from master, [380ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [383ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1846377997(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6e381ae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1777092961 after retry RPC@2021429893) -> (continuation of Deferred@829487004 after retry RPC@763671437) -> (continuation of Deferred@357006246 after retry RPC@500673112) -> (continuation of Deferred@1741790519 after retry RPC@1207205631), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c39a70e -> wakeup thread Time-limited test -> (continuation of Deferred@1777092961 after retry RPC@2021429893) -> (continuation of Deferred@829487004 after retry RPC@763671437) -> (continuation of Deferred@357006246 after retry RPC@500673112) -> (continuation of Deferred@1741790519 after retry RPC@1207205631))) after lookup
20:04:06.832 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] connecting to peer
20:04:06.833 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.833 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] cleaning up while in state CONNECTING due to: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:06.833 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045
20:04:07.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.743593  6960 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 9 FOLLOWER]: Starting pre-election (detected failure of leader 533c3ef5bc9342ac84c55607e02629a8)
20:04:07.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.743724  6960 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 9 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:07.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.744712  6960 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 10 pre-election: Requested pre-vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:04:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:07.745334  6653 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111)
20:04:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:07.746055  6653 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 10 pre-election: RPC error from VoteRequest() call to peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111)
20:04:07.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.748382  6900 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "2fb2124f5c8a4a78b46d0db63c580371" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "f37d275110584bbb8c3b91cf61249854" is_pre_election: true
20:04:07.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.748649  6900 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 8 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2fb2124f5c8a4a78b46d0db63c580371 in term 8.
20:04:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.748993  6654 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 10 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, f37d275110584bbb8c3b91cf61249854; no voters: 533c3ef5bc9342ac84c55607e02629a8
20:04:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.749202  6960 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 9 FOLLOWER]: Leader pre-election won for term 10
20:04:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.749258  6960 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 9 FOLLOWER]: Starting leader election (detected failure of leader 533c3ef5bc9342ac84c55607e02629a8)
20:04:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.749305  6960 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 9 FOLLOWER]: Advancing to term 10
20:04:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.750053  6960 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 10 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.750350  6960 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 10 election: Requested vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), f37d275110584bbb8c3b91cf61249854 (127.4.172.67:33409)
20:04:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.750748  6900 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "2fb2124f5c8a4a78b46d0db63c580371" candidate_term: 10 candidate_status { last_received { term: 9 index: 36 } } ignore_live_leader: false dest_uuid: "f37d275110584bbb8c3b91cf61249854"
20:04:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.750880  6900 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 8 FOLLOWER]: Advancing to term 10
20:04:07.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:07.751300  6653 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 10 election: RPC error from VoteRequest() call to peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111)
20:04:07.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.752652  6900 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 10 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2fb2124f5c8a4a78b46d0db63c580371 in term 10.
20:04:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.752959  6654 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [CANDIDATE]: Term 10 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2fb2124f5c8a4a78b46d0db63c580371, f37d275110584bbb8c3b91cf61249854; no voters: 533c3ef5bc9342ac84c55607e02629a8
20:04:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.753122  6960 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 10 FOLLOWER]: Leader election won for term 10
20:04:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.753310  6960 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [term 10 LEADER]: Becoming Leader. State: Replica: 2fb2124f5c8a4a78b46d0db63c580371, State: Running, Role: LEADER
20:04:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.753582  6960 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 36, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 10, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:07.756150  4972 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 reported cstate change: term changed from 9 to 10, leader changed from 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65) to 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66). New cstate: current_term: 10 leader_uuid: "2fb2124f5c8a4a78b46d0db63c580371" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: UNKNOWN } } }
20:04:08.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:08.147488  6653 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 -> Peer 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045): Couldn't send request to peer 533c3ef5bc9342ac84c55607e02629a8. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.65:46045: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:08.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.212239  6900 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 10 FOLLOWER]: Refusing update from remote peer 2fb2124f5c8a4a78b46d0db63c580371: Log matching property violated. Preceding OpId in replica: term: 8 index: 32. Preceding OpId from leader: term: 10 index: 37. (index mismatch)
20:04:08.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.212908  6960 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 37, Last known committed idx: 32, Time since last communication: 0.000s
20:04:08.251 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:08.252 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:08.252 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=12, sequence_id=28, TimeoutTracker(timeout=30000, elapsed=1804), Traces: [0ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [1ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [20ms] refreshing cache from master, [20ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [22ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [23ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [25ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [25ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [44ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [46ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [46ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [80ms] refreshing cache from master, [81ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [83ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [84ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [85ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [85ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [120ms] refreshing cache from master, [120ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [122ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [123ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [125ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [125ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [380ms] refreshing cache from master, [380ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [383ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [384ms] sending RPC to server 533c3ef5bc9342ac84c55607e02629a8, [385ms] received response from server 533c3ef5bc9342ac84c55607e02629a8: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [385ms] delaying RPC due to: Network error: Failed to connect to peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045): Connection refused: /127.4.172.65:46045, [1800ms] refreshing cache from master, [1801ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [1803ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@1846377997(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@6e381ae9 -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@1777092961 after retry RPC@2021429893) -> (continuation of Deferred@829487004 after retry RPC@763671437) -> (continuation of Deferred@357006246 after retry RPC@500673112) -> (continuation of Deferred@1741790519 after retry RPC@1207205631) -> (continuation of Deferred@1256921463 after retry RPC@1334202130), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@5c39a70e -> wakeup thread Time-limited test -> (continuation of Deferred@1777092961 after retry RPC@2021429893) -> (continuation of Deferred@829487004 after retry RPC@763671437) -> (continuation of Deferred@357006246 after retry RPC@500673112) -> (continuation of Deferred@1741790519 after retry RPC@1207205631) -> (continuation of Deferred@1256921463 after retry RPC@1334202130))) after lookup
20:04:08.253 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:08.254 [DEBUG - kudu-nio-3] (Connection.java:757) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] Successfully connected to peer
20:04:08.266 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0xc3a4a9aa, L:/127.0.0.1:40562 - R:/127.4.172.66:44141] using TOKEN/null
20:04:08.280 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.4.172.65:46045
20:04:08.280 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 0
  }
}

20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.281317  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/logs
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:08.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.65:46045
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.65
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43765
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.65
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:08.380685  6987 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:08.380877  6987 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:08.380928  6987 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:08.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:08.384122  6987 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:08.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:08.384219  6987 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.65
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.387862  6987 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.65:46045
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.65
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43765
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.6987
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.65
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/logs
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:04:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:04:08.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.388883  6987 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:08.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.389883  6987 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:08.395809  6992 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:08.395828  6995 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:08.395809  6993 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.395977  6987 server_base.cc:1061] running on GCE node
20:04:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.396512  6987 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.397053  6987 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.398195  6987 hybrid_clock.cc:648] HybridClock initialized: now 1779134648398183 us; error 27 us; skew 500 ppm
20:04:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.399868  6987 webserver.cc:533] Webserver started at http://127.4.172.65:43765/ using document root <none> and password file <none>
20:04:08.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.400408  6987 fs_manager.cc:362] Metadata directory not provided
20:04:08.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.400465  6987 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.403855  6987 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.002s	sys 0.002s
20:04:08.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.405915  7001 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:08.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.406831  6987 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.001s
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.406927  6987 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:38 on dist-test-slave-2cm3"
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.407313  6987 fs_report.cc:389] FS layout report
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/wal
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/data
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:08.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.418253  6987 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.418957  6987 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.419126  6987 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.419683  6987 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.420926  7008 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
20:04:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.424926  6987 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.424988  6987 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
20:04:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.425050  6987 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.426640  6987 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.426689  6987 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.002s	sys 0.000s
20:04:08.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.426911  7008 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Bootstrap starting.
20:04:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.437007  6987 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.65:46045
20:04:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.437047  7115 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.65:46045 every 8 connection(s)
20:04:08.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.437965  6987 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-0/data/info.pb
20:04:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.443272  7008 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Log is configured to *not* fsync() on all Append() calls
20:04:08.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.446391  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 6987
20:04:08.446 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:08.455 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 30 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:08.456 [INFO - Time-limited test] (MiniKuduCluster.java:475) Killing tablet server 127.4.172.66:44141
20:04:08.456 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:04:08.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.456665  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 6636
20:04:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.459187  7118 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:04:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.459461  7118 heartbeater.cc:461] Registering TS with master...
20:04:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.460088  7118 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:04:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.462092  4900 ts_manager.cc:194] Re-registered known tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:04:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.464529  7116 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:04:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.464619  7116 heartbeater.cc:461] Registering TS with master...
20:04:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.464821  7116 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:04:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.465672  4971 ts_manager.cc:194] Re-registered known tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:04:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.466626  4971 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.65:42375
20:04:08.466 [DEBUG - kudu-nio-3] (Connection.java:239) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] handling channelInactive
20:04:08.467 [DEBUG - kudu-nio-3] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state READY due to: connection closed
20:04:08.467 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:08.468 [DEBUG - Time-limited test] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:08.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.468303  7117 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:04:08.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.468389  7117 heartbeater.cc:461] Registering TS with master...
20:04:08.469 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.469 [DEBUG - kudu-nio-4] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.469 [INFO - kudu-nio-4] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.468567  7117 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:04:08.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.468721  7008 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Bootstrap replayed 1/1 log segments. Stats: ops{read=36 overwritten=0 applied=36 ignored=0} inserts{seen=27 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:08.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.469053  7008 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Bootstrap complete.
20:04:08.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.469548  4826 ts_manager.cc:194] Re-registered known tserver with Master: 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045)
20:04:08.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.469851  7008 ts_tablet_manager.cc:1403] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Time spent bootstrapping tablet: real 0.043s	user 0.031s	sys 0.009s
20:04:08.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.471915  7008 raft_consensus.cc:359] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 9 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:08.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.472246  7008 raft_consensus.cc:740] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 9 FOLLOWER]: Becoming Follower/Learner. State: Replica: 533c3ef5bc9342ac84c55607e02629a8, State: Initialized, Role: FOLLOWER
20:04:08.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.472674  7008 consensus_queue.cc:260] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 36, Last appended: 9.36, Last appended by leader: 36, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.473306  7116 heartbeater.cc:499] Master 127.4.172.124:33635 was elected leader, sending a full tablet report...
20:04:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:08.473524  7008 ts_tablet_manager.cc:1434] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8: Time spent starting tablet: real 0.004s	user 0.006s	sys 0.000s
20:04:08.490 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:08.490 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:08.491 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=2, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=23), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [2ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2055254368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@7039ffd -> passthrough -> wakeup thread Time-limited test, errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29fa7d00 -> wakeup thread Time-limited test)) after lookup
20:04:08.491 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:08.492 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.492 [DEBUG - kudu-nio-5] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.492 [INFO - kudu-nio-5] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.510 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:08.511 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:08.511 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=4, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=43), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [2ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [25ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [25ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2055254368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@7039ffd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29fa7d00 -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035))) after lookup
20:04:08.512 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:08.512 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.512 [DEBUG - kudu-nio-6] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.512 [INFO - kudu-nio-6] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.531 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:08.531 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:08.531 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=6, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=64), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [2ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [25ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [25ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [44ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [45ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [46ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [64ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2055254368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@7039ffd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035) -> (continuation of Deferred@1563245764 after retry RPC@764526279), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29fa7d00 -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035) -> (continuation of Deferred@1563245764 after retry RPC@764526279))) after lookup
20:04:08.532 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:08.533 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.533 [DEBUG - kudu-nio-7] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.533 [INFO - kudu-nio-7] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.652 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:08.652 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:08.652 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=8, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=184), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [2ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [25ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [25ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [44ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [45ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [46ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [64ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [65ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [66ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [66ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [182ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [184ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2055254368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@7039ffd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035) -> (continuation of Deferred@1563245764 after retry RPC@764526279) -> (continuation of Deferred@2144285739 after retry RPC@1602804371), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29fa7d00 -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035) -> (continuation of Deferred@1563245764 after retry RPC@764526279) -> (continuation of Deferred@2144285739 after retry RPC@1602804371))) after lookup
20:04:08.653 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:08.654 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.654 [DEBUG - kudu-nio-0] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.654 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.971 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:08.971 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:08.972 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=10, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=504), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [2ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [25ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [25ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [44ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [45ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [46ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [64ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [65ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [66ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [66ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [182ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [184ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [186ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [187ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [187ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [501ms] refreshing cache from master, [501ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [503ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2055254368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@7039ffd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035) -> (continuation of Deferred@1563245764 after retry RPC@764526279) -> (continuation of Deferred@2144285739 after retry RPC@1602804371) -> (continuation of Deferred@1822183224 after retry RPC@1977179258), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29fa7d00 -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035) -> (continuation of Deferred@1563245764 after retry RPC@764526279) -> (continuation of Deferred@2144285739 after retry RPC@1602804371) -> (continuation of Deferred@1822183224 after retry RPC@1977179258))) after lookup
20:04:08.972 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:08.973 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.973 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:08.974 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:09.270 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:09.271 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)[L],533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)]}]
20:04:09.271 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=12, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=803), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [2ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [25ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [25ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [44ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [45ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [46ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [64ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [65ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [66ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [66ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [182ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [184ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [186ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [187ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [187ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [501ms] refreshing cache from master, [501ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [503ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [505ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [507ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [507ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [801ms] refreshing cache from master, [801ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [803ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2055254368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@7039ffd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035) -> (continuation of Deferred@1563245764 after retry RPC@764526279) -> (continuation of Deferred@2144285739 after retry RPC@1602804371) -> (continuation of Deferred@1822183224 after retry RPC@1977179258) -> (continuation of Deferred@1980222399 after retry RPC@1233286025), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29fa7d00 -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035) -> (continuation of Deferred@1563245764 after retry RPC@764526279) -> (continuation of Deferred@2144285739 after retry RPC@1602804371) -> (continuation of Deferred@1822183224 after retry RPC@1977179258) -> (continuation of Deferred@1980222399 after retry RPC@1233286025))) after lookup
20:04:09.271 [DEBUG - kudu-nio-2] (Connection.java:748) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] connecting to peer
20:04:09.272 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:09.272 [DEBUG - kudu-nio-2] (Connection.java:709) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] cleaning up while in state CONNECTING due to: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:09.272 [INFO - kudu-nio-2] (AsyncKuduClient.java:2355) Invalidating location 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141) for tablet a42d6caf61bb47e19a3389753a1059a1: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141
20:04:09.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.780496  7129 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 10 FOLLOWER]: Starting pre-election (detected failure of leader 2fb2124f5c8a4a78b46d0db63c580371)
20:04:09.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.780629  7129 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 10 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:09.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.781651  7129 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 11 pre-election: Requested pre-vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:04:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:09.782568  6834 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:04:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:09.783390  6834 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 11 pre-election: RPC error from VoteRequest() call to peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:04:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.786173  7070 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "f37d275110584bbb8c3b91cf61249854" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "533c3ef5bc9342ac84c55607e02629a8" is_pre_election: true
20:04:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.786458  7070 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 9 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f37d275110584bbb8c3b91cf61249854 in term 9.
20:04:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.787184  6834 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 11 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 533c3ef5bc9342ac84c55607e02629a8, f37d275110584bbb8c3b91cf61249854; no voters: 2fb2124f5c8a4a78b46d0db63c580371
20:04:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.787449  7129 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 10 FOLLOWER]: Leader pre-election won for term 11
20:04:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.787523  7129 raft_consensus.cc:493] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 10 FOLLOWER]: Starting leader election (detected failure of leader 2fb2124f5c8a4a78b46d0db63c580371)
20:04:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.787580  7129 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 10 FOLLOWER]: Advancing to term 11
20:04:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.788602  7129 raft_consensus.cc:515] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 11 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.788962  7129 leader_election.cc:290] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 11 election: Requested vote from peers 533c3ef5bc9342ac84c55607e02629a8 (127.4.172.65:46045), 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:04:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.789325  7070 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "a42d6caf61bb47e19a3389753a1059a1" candidate_uuid: "f37d275110584bbb8c3b91cf61249854" candidate_term: 11 candidate_status { last_received { term: 10 index: 40 } } ignore_live_leader: false dest_uuid: "533c3ef5bc9342ac84c55607e02629a8"
20:04:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.789429  7070 raft_consensus.cc:3060] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 9 FOLLOWER]: Advancing to term 11
20:04:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:09.789999  6834 leader_election.cc:336] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 11 election: RPC error from VoteRequest() call to peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111)
20:04:09.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.791146  7070 raft_consensus.cc:2468] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 11 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f37d275110584bbb8c3b91cf61249854 in term 11.
20:04:09.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.791438  6834 leader_election.cc:304] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [CANDIDATE]: Term 11 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 533c3ef5bc9342ac84c55607e02629a8, f37d275110584bbb8c3b91cf61249854; no voters: 2fb2124f5c8a4a78b46d0db63c580371
20:04:09.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.791582  7129 raft_consensus.cc:2804] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 11 FOLLOWER]: Leader election won for term 11
20:04:09.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.791764  7129 raft_consensus.cc:697] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [term 11 LEADER]: Becoming Leader. State: Replica: f37d275110584bbb8c3b91cf61249854, State: Running, Role: LEADER
20:04:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.792001  7129 consensus_queue.cc:237] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 10.40, Last appended by leader: 40, Current term: 11, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } }
20:04:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:09.794242  4971 catalog_manager.cc:5696] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 reported cstate change: term changed from 10 to 11, leader changed from 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66) to f37d275110584bbb8c3b91cf61249854 (127.4.172.67). New cstate: current_term: 11 leader_uuid: "f37d275110584bbb8c3b91cf61249854" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2fb2124f5c8a4a78b46d0db63c580371" member_type: VOTER last_known_addr { host: "127.4.172.66" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f37d275110584bbb8c3b91cf61249854" member_type: VOTER last_known_addr { host: "127.4.172.67" port: 33409 } health_report { overall_health: HEALTHY } } }
20:04:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:10.286863  7070 raft_consensus.cc:1275] T a42d6caf61bb47e19a3389753a1059a1 P 533c3ef5bc9342ac84c55607e02629a8 [term 11 FOLLOWER]: Refusing update from remote peer f37d275110584bbb8c3b91cf61249854: Log matching property violated. Preceding OpId in replica: term: 9 index: 36. Preceding OpId from leader: term: 11 index: 41. (index mismatch)
20:04:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:10.287513  7134 consensus_queue.cc:1048] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 [LEADER]: Connected to new peer: Peer: permanent_uuid: "533c3ef5bc9342ac84c55607e02629a8" member_type: VOTER last_known_addr { host: "127.4.172.65" port: 46045 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 36, Time since last communication: 0.000s
20:04:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:10.296665  6834 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 -> Peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Couldn't send request to peer 2fb2124f5c8a4a78b46d0db63c580371. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:12.599999  6834 consensus_peers.cc:597] T a42d6caf61bb47e19a3389753a1059a1 P f37d275110584bbb8c3b91cf61249854 -> Peer 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141): Couldn't send request to peer 2fb2124f5c8a4a78b46d0db63c580371. Status: Network error: Client connection negotiation failed: client connection to 127.4.172.66:44141: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
20:04:13.152 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2538) Learned about tablet a42d6caf61bb47e19a3389753a1059a1 for table 'org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735' with partition [<start>, <end>)
20:04:13.152 [DEBUG - kudu-nio-2] (TableLocationsCache.java:192) Discovered table locations:	[Tablet{lowerBoundPartitionKey=0x, upperBoundPartitionKey=0x, ttl=299999, tablet=a42d6caf61bb47e19a3389753a1059a1@[2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141),533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045),f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)[L]]}]
20:04:13.153 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=a42d6caf61bb47e19a3389753a1059a1, attempt=14, sequence_id=31, TimeoutTracker(timeout=30000, elapsed=4685), Traces: [0ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [2ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [2ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [21ms] refreshing cache from master, [21ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [23ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [24ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [25ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [25ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [41ms] refreshing cache from master, [41ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [43ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [44ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [45ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [46ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [62ms] refreshing cache from master, [62ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [64ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [65ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [66ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [66ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [182ms] refreshing cache from master, [182ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [184ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [186ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [187ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [187ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [501ms] refreshing cache from master, [501ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [503ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [505ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [507ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [507ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [801ms] refreshing cache from master, [801ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [803ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, [804ms] sending RPC to server 2fb2124f5c8a4a78b46d0db63c580371, [805ms] received response from server 2fb2124f5c8a4a78b46d0db63c580371: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [805ms] delaying RPC due to: Network error: Failed to connect to peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141): Connection refused: /127.4.172.66:44141, [4682ms] refreshing cache from master, [4682ms] Sub RPC GetTableLocations: sending RPC to server master-127.4.172.124:33635, [4685ms] Sub RPC GetTableLocations: received response from server master-127.4.172.124:33635: OK, deferred=Deferred@2055254368(state=PENDING, result=null, callback=org.apache.kudu.client.AsyncKuduSession$$Lambda$235/0x00007f29c4336510@7039ffd -> passthrough -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035) -> (continuation of Deferred@1563245764 after retry RPC@764526279) -> (continuation of Deferred@2144285739 after retry RPC@1602804371) -> (continuation of Deferred@1822183224 after retry RPC@1977179258) -> (continuation of Deferred@1980222399 after retry RPC@1233286025) -> (continuation of Deferred@243822880 after retry RPC@1797689865), errback=passthrough -> org.apache.kudu.client.AsyncKuduSession$SingleOperationErrCallback@29fa7d00 -> wakeup thread Time-limited test -> (continuation of Deferred@877638807 after retry RPC@1319624035) -> (continuation of Deferred@1563245764 after retry RPC@764526279) -> (continuation of Deferred@2144285739 after retry RPC@1602804371) -> (continuation of Deferred@1822183224 after retry RPC@1977179258) -> (continuation of Deferred@1980222399 after retry RPC@1233286025) -> (continuation of Deferred@243822880 after retry RPC@1797689865))) after lookup
20:04:13.153 [DEBUG - kudu-nio-2] (Connection.java:748) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] connecting to peer
20:04:13.154 [DEBUG - kudu-nio-3] (Connection.java:757) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] Successfully connected to peer
20:04:13.168 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x93417a45, L:/127.0.0.1:56080 - R:/127.4.172.67:33409] using TOKEN/null
20:04:13.183 [INFO - Time-limited test] (MiniKuduCluster.java:456) Starting tablet server 127.4.172.66:44141
20:04:13.183 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: TSERVER
    index: 1
  }
}

20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.184238  4785 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskswF7jE/build/debug/bin/kudu
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/logs
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.66:44141
20:04:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.66
20:04:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:04:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41487
20:04:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.66
20:04:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:04:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:04:13.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:13.288686  7166 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:13.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:13.288892  7166 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:13.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:13.288937  7166 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:13.292260  7166 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:13.292373  7166 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.172.66
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.296182  7166 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.172.84:38663
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.172.66:44141
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.172.66
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41487
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.172.126:44523,127.4.172.125:33165,127.4.172.124:33635
20:04:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.7166
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.172.66
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/logs
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 66827b15b158c37363c37baa160f29a0ca1f1e63
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 May 2026 19:43:15 UTC on bdcb31816ec0
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11960
20:04:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.297463  7166 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:13.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.298944  7166 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:13.306874  7174 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:13.306972  7171 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.307324  7166 server_base.cc:1061] running on GCE node
20:04:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260518 20:04:13.306851  7172 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.307842  7166 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.308350  7166 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.309502  7166 hybrid_clock.cc:648] HybridClock initialized: now 1779134653309481 us; error 38 us; skew 500 ppm
20:04:13.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.311291  7166 webserver.cc:533] Webserver started at http://127.4.172.66:41487/ using document root <none> and password file <none>
20:04:13.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.311914  7166 fs_manager.cc:362] Metadata directory not provided
20:04:13.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.311975  7166 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:13.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.315335  7166 fs_manager.cc:714] Time spent opening directory manager: real 0.002s	user 0.003s	sys 0.000s
20:04:13.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.317203  7180 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
20:04:13.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.318176  7166 fs_manager.cc:730] Time spent opening block manager: real 0.002s	user 0.002s	sys 0.001s
20:04:13.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.318280  7166 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data,/tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:04:13.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2fb2124f5c8a4a78b46d0db63c580371"
20:04:13.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-18 20:03:39 on dist-test-slave-2cm3"
20:04:13.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.318728  7166 fs_report.cc:389] FS layout report
20:04:13.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/wal
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/data
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:13.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.335879  7166 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
20:04:13.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.336683  7166 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:13.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.336872  7166 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:13.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.337496  7166 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:13.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.338897  7187 ts_tablet_manager.cc:542] Loading tablet metadata (0/1 complete)
20:04:13.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.342211  7166 ts_tablet_manager.cc:585] Loaded tablet metadata (1 total tablets, 1 live tablets)
20:04:13.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.342269  7166 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.004s	user 0.000s	sys 0.000s
20:04:13.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.342334  7166 ts_tablet_manager.cc:600] Registering tablets (0/1 complete)
20:04:13.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.343926  7166 ts_tablet_manager.cc:616] Registered 1 tablets
20:04:13.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.343971  7166 ts_tablet_manager.cc:595] Time spent register tablets: real 0.002s	user 0.000s	sys 0.000s
20:04:13.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.344362  7187 tablet_bootstrap.cc:492] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Bootstrap starting.
20:04:13.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.355140  7166 rpc_server.cc:307] RPC server started. Bound to: 127.4.172.66:44141
20:04:13.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.356155  7166 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskswF7jE/test-tmp/mini-kudu-cluster6399550616659062952/ts-1/data/info.pb
20:04:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.357008  7295 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.172.66:44141 every 8 connection(s)
20:04:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.359647  4785 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskswF7jE/build/debug/bin/kudu as pid 7166
20:04:13.360 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
20:04:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.364588  7187 log.cc:826] T a42d6caf61bb47e19a3389753a1059a1 P 2fb2124f5c8a4a78b46d0db63c580371: Log is configured to *not* fsync() on all Append() calls
20:04:13.368 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:13.372 [DEBUG - kudu-nio-3] (AsyncKuduScanner.java:1370) AsyncKuduScanner$Response(scannerId = "", data = RowwiseRowResultIterator for 33 rows, more = false) for scanner KuduScanner(table=org.apache.kudu.client.TestMultipleLeaderFailover-1779134619735, tablet=a42d6caf61bb47e19a3389753a1059a1, scannerId=null, scanRequestTimeout=30000, startPrimaryKey=<start>, endPrimaryKey=<end>, lastPrimaryKey=<last>)
20:04:13.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.377203  7296 heartbeater.cc:344] Connected to a master server at 127.4.172.124:33635
20:04:13.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.377473  7296 heartbeater.cc:461] Registering TS with master...
20:04:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.378104  7296 heartbeater.cc:507] Master 127.4.172.124:33635 requested a full tablet report, sending...
20:04:13.378 [DEBUG - main] (Connection.java:575) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] disconnecting while in state READY
20:04:13.379 [DEBUG - main] (Connection.java:575) [peer master-127.4.172.124:33635(127.4.172.124:33635)] disconnecting while in state READY
20:04:13.379 [DEBUG - main] (Connection.java:575) [peer 2fb2124f5c8a4a78b46d0db63c580371(127.4.172.66:44141)] disconnecting while in state TERMINATED
20:04:13.379 [DEBUG - main] (Connection.java:575) [peer 533c3ef5bc9342ac84c55607e02629a8(127.4.172.65:46045)] disconnecting while in state TERMINATED
20:04:13.379 [DEBUG - main] (Connection.java:575) [peer master-127.4.172.126:44523(127.4.172.126:44523)] disconnecting while in state READY
20:04:13.380 [DEBUG - main] (Connection.java:575) [peer master-127.4.172.125:33165(127.4.172.125:33165)] disconnecting while in state READY
20:04:13.382 [DEBUG - kudu-nio-3] (Connection.java:239) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] handling channelInactive
20:04:13.382 [DEBUG - kudu-nio-3] (Connection.java:709) [peer f37d275110584bbb8c3b91cf61249854(127.4.172.67:33409)] cleaning up while in state READY due to: connection closed
20:04:13.382 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.172.126:44523(127.4.172.126:44523)] handling channelInactive
20:04:13.382 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.172.126:44523(127.4.172.126:44523)] cleaning up while in state READY due to: connection closed
20:04:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.383663  4971 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:04:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.384428  4971 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.4.172.66:48085
20:04:13.384 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.4.172.124:33635(127.4.172.124:33635)] handling channelInactive
20:04:13.384 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.4.172.124:33635(127.4.172.124:33635)] cleaning up while in state READY due to: connection closed
20:04:13.386 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:04:13.387 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.4.172.125:33165(127.4.172.125:33165)] handling channelInactive
20:04:13.388 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.172.125:33165(127.4.172.125:33165)] cleaning up while in state READY due to: connection closed
20:04:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.388347  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 6987
20:04:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.392269  7297 heartbeater.cc:344] Connected to a master server at 127.4.172.126:44523
20:04:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.392398  7297 heartbeater.cc:461] Registering TS with master...
20:04:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.392504  7298 heartbeater.cc:344] Connected to a master server at 127.4.172.125:33165
20:04:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.392596  7298 heartbeater.cc:461] Registering TS with master...
20:04:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.392649  7297 heartbeater.cc:507] Master 127.4.172.126:44523 requested a full tablet report, sending...
20:04:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.392725  7298 heartbeater.cc:507] Master 127.4.172.125:33165 requested a full tablet report, sending...
20:04:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.393700  4827 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:04:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.393702  4900 ts_manager.cc:194] Re-registered known tserver with Master: 2fb2124f5c8a4a78b46d0db63c580371 (127.4.172.66:44141)
20:04:13.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.398645  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 7166
20:04:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.404969  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 6817
20:04:13.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.410931  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 4796
20:04:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.416500  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 4866
20:04:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260518 20:04:13.422233  4785 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskswF7jE/build/debug/bin/kudu with pid 4936
20:04:13.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-18T20:04:13Z chronyd exiting

Time: 36.515

OK (1 test)

